KleidiAI Coverage Report

Directory: ./
File: test/tests/matmul_clamp_f32_qsi8d32p_qai4c32p_test.cpp
Date: 2025-10-20 13:18:31
Coverage Exec Excl Total
Lines: 98.6% 142 0 144
Functions: 100.0% 15 0 15
Branches: 41.8% 234 0 560
Line Branch Exec Source
1 //
2 // SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its affiliates <open-source-office@arm.com>
3 //
4 // SPDX-License-Identifier: Apache-2.0
5 //
6
7 #include <gtest/gtest.h>
8
9 #include <array>
10 #include <cstddef>
11 #include <cstdint>
12 #include <cstdlib>
13 #include <sstream>
14 #include <string>
15 #include <tuple>
16
17 #include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qai4c32p/kai_matmul_clamp_f32_qsi8d32p1vlx4_qai4c32p4vlx4_1vlx4vl_sme2_mopa.h"
18 #include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qai4c32p/kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p4vlx4_1x4vl_sme2_dot.h"
19 #include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qai4c32p/kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p4x4_1x4_neon_dotprod.h"
20 #include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qai4c32p/kai_matmul_clamp_f32_qsi8d32p1x8_qai4c32p4x8_1x4_neon_dotprod.h"
21 #include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qai4c32p/kai_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod.h"
22 #include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qai4c32p/kai_matmul_clamp_f32_qsi8d32p4x8_qai4c32p4x8_8x4_neon_i8mm.h"
23 #include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qai4c32p/kai_matmul_clamp_f32_qsi8d32p_qai4c32p_interface.h"
24 #include "kai/ukernels/matmul/pack/kai_lhs_quant_pack_qsi8d32pscalef32_f32_neon.h"
25 #include "kai/ukernels/matmul/pack/kai_rhs_pack_nxk_qai4c32p_qau4c32s0s1_f32_f32_f32_neon.h"
26 #include "kai/ukernels/matmul/pack/kai_rhs_pack_nxk_qai4c32ps1s0nrx4_qau4c32s0s1_f32_f32_f32_neon.h"
27 #include "kai/ukernels/matmul/pack/kai_rhs_pack_nxk_qai4c32ps1s0nrx4_qau4c32s1s0_f32_f32_f32_neon.h"
28 #include "test/common/buffer.hpp"
29 #include "test/common/compare.hpp"
30 #include "test/common/cpu_info.hpp"
31 #include "test/common/int4.hpp"
32 #include "test/common/matmul_test_common.hpp"
33 #include "test/common/matrix_portion.hpp"
34 #include "test/common/memory.hpp"
35 #include "test/common/round.hpp"
36 #include "test/common/test_suite.hpp"
37 #include "test/reference/cast.hpp"
38 #include "test/reference/clamp.hpp"
39 #include "test/reference/fill.hpp"
40 #include "test/reference/matmul.hpp"
41 #include "test/reference/pack.hpp"
42 #include "test/reference/quantize.hpp"
43
44 namespace kai::test {
45 // Interface for the LHS and RHS packed size and packing micro-kernels
46 using kai_get_lhs_packed_size_func_t = decltype(&kai_get_lhs_packed_size_lhs_quant_pack_qsi8d32pscalef32_f32_neon);
47 using kai_get_rhs_packed_size_func_t =
48 decltype(&kai_get_rhs_packed_size_rhs_pack_nxk_qai4c32p_qau4c32s0s1_f32_f32_f32_neon);
49 using kai_get_lhs_packed_offset_func_t = decltype(&kai_get_lhs_packed_offset_lhs_quant_pack_qsi8d32pscalef32_f32_neon);
50 using kai_get_rhs_packed_offset_func_t =
51 decltype(&kai_get_rhs_packed_offset_rhs_pack_nxk_qai4c32p_qau4c32s0s1_f32_f32_f32_neon);
52 using kai_get_lhs_offset_func_t = decltype(&kai_get_lhs_offset_lhs_quant_pack_qsi8d32pscalef32_f32_neon);
53 using kai_get_rhs_offset_func_t = decltype(&kai_get_rhs_offset_rhs_pack_nxk_qai4c32p_qau4c32s0s1_f32_f32_f32_neon);
54 using kai_run_lhs_pack_func_t = decltype(&kai_run_lhs_quant_pack_qsi8d32pscalef32_f32_neon);
55 using kai_run_rhs_pack_func_t = decltype(&kai_run_rhs_pack_nxk_qai4c32p_qau4c32s0s1_f32_f32_f32_neon);
56
57 // Micro-kernel interface
58 struct kai_qai4c32p_pack_functions {
59 kai_get_rhs_packed_size_func_t packed_size;
60 kai_get_rhs_packed_offset_func_t get_packed_offset;
61 kai_get_rhs_offset_func_t get_offset;
62 kai_run_rhs_pack_func_t run_pack;
63 };
64
65 struct kai_qsi8d32p_pack_functions {
66 kai_get_lhs_packed_size_func_t packed_size;
67 kai_get_lhs_packed_offset_func_t get_packed_offset;
68 kai_get_lhs_offset_func_t get_offset;
69 kai_run_lhs_pack_func_t run_pack;
70 };
71
72 static const std::array<
73 UkernelMatmulPackVariant<
74 kai_matmul_clamp_f32_qsi8d32p_qai4c32p_ukernel, kai_qsi8d32p_pack_functions, kai_qai4c32p_pack_functions>,
75 8>
76 variants_kai_matmul_clamp_f32_qsi8d32p_qai4c32p = {
77 {UKERNEL_MATMUL_PACK_VARIANT(
78 clamp_f32_qsi8d32p1x8_qai4c32p4x8_1x4_neon_dotprod, cpu_has_dotprod,
79 lhs_quant_pack_qsi8d32pscalef32_f32_neon, rhs_pack_nxk_qai4c32p_qau4c32s0s1_f32_f32_f32_neon, true),
80 UKERNEL_MATMUL_PACK_VARIANT(
81 clamp_f32_qsi8d32p4x8_qai4c32p4x8_8x4_neon_i8mm, cpu_has_i8mm, lhs_quant_pack_qsi8d32pscalef32_f32_neon,
82 rhs_pack_nxk_qai4c32p_qau4c32s0s1_f32_f32_f32_neon, true),
83 UKERNEL_MATMUL_PACK_VARIANT(
84 clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod, cpu_has_dotprod,
85 lhs_quant_pack_qsi8d32pscalef32_f32_neon, rhs_pack_nxk_qai4c32p_qau4c32s0s1_f32_f32_f32_neon, true),
86 UKERNEL_MATMUL_PACK_VARIANT(
87 clamp_f32_qsi8d32p1x4_qai4c32p4x4_1x4_neon_dotprod, cpu_has_dotprod,
88 lhs_quant_pack_qsi8d32pscalef32_f32_neon, rhs_pack_nxk_qai4c32p_qau4c32s0s1_f32_f32_f32_neon, true),
89 UKERNEL_MATMUL_PACK_VARIANT(
90 clamp_f32_qsi8d32p1x4_qai4c32p4vlx4_1x4vl_sme2_dot, cpu_has_sme2, lhs_quant_pack_qsi8d32pscalef32_f32_neon,
91 rhs_pack_nxk_qai4c32ps1s0nrx4_qau4c32s1s0_f32_f32_f32_neon, false),
92 UKERNEL_MATMUL_PACK_VARIANT(
93 clamp_f32_qsi8d32p1vlx4_qai4c32p4vlx4_1vlx4vl_sme2_mopa, cpu_has_sme2,
94 lhs_quant_pack_qsi8d32pscalef32_f32_neon, rhs_pack_nxk_qai4c32ps1s0nrx4_qau4c32s1s0_f32_f32_f32_neon,
95 false),
96 UKERNEL_MATMUL_PACK_VARIANT(
97 clamp_f32_qsi8d32p1x4_qai4c32p4vlx4_1x4vl_sme2_dot, cpu_has_sme2, lhs_quant_pack_qsi8d32pscalef32_f32_neon,
98 rhs_pack_nxk_qai4c32ps1s0nrx4_qau4c32s0s1_f32_f32_f32_neon, true),
99 UKERNEL_MATMUL_PACK_VARIANT(
100 clamp_f32_qsi8d32p1vlx4_qai4c32p4vlx4_1vlx4vl_sme2_mopa, cpu_has_sme2,
101 lhs_quant_pack_qsi8d32pscalef32_f32_neon, rhs_pack_nxk_qai4c32ps1s0nrx4_qau4c32s0s1_f32_f32_f32_neon,
102 true)}};
103
104 // Executes the LHS packing micro-kernel.
105 10904 static inline std::tuple<Buffer, size_t> pack_lhs_qsi8d32p(
106 const kai_qsi8d32p_pack_functions& pack_interface, size_t M, size_t K, size_t bl, size_t mr, size_t kr, size_t sr,
107 const Buffer& lhs_values_qsi8, size_t stride, size_t rect_start_row, size_t rect_height) {
108 10904 const auto imp_packed_lhs_size = pack_interface.packed_size(M, K, bl, mr, kr, sr);
109 10904 Buffer imp_packed_lhs(imp_packed_lhs_size, 0);
110
111
1/2
✓ Branch 0 taken 10904 times.
✗ Branch 1 not taken.
 10904 auto lhs_offset = pack_interface.get_offset(rect_start_row, stride);
112
1/2
✓ Branch 0 taken 10904 times.
✗ Branch 1 not taken.
 10904 auto lhs_packed_offset = pack_interface.get_packed_offset(rect_start_row, K, bl, mr, kr, sr);
113
114
2/4
✓ Branch 0 taken 10904 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 10904 times.
✗ Branch 3 not taken.
 21808 pack_interface.run_pack(
115 10904 rect_height, K, bl, mr, kr, sr, 0, reinterpret_cast<const float*>(lhs_values_qsi8.data() + lhs_offset), stride,
116 10904 imp_packed_lhs.data() + lhs_packed_offset);
117
118 10904 return {std::move(imp_packed_lhs), lhs_packed_offset};
119 10904 }
120
121 // Executes the RHS packing micro-kernel.
122 2616 static inline std::tuple<Buffer, size_t> pack_rhs_qai4c32p(
123 const kai_qai4c32p_pack_functions& pack_interface, size_t N, size_t K, size_t bl, size_t nr, size_t kr, size_t sr,
124 const Buffer& rhs_values_qai4, const bool has_bias, const Buffer& biases, const Buffer& rhs_scales,
125 const Buffer& rhs_zp, bool s0s1_input, size_t rect_start_row) {
126 // Cast to unsigned int
127 2616 auto rhs_qau4s1s0 = cast_qsu4_qsi4(rhs_values_qai4.data(), N * K);
128
129
1/2
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
 2616 const auto imp_packed_rhs_size = pack_interface.packed_size(N, K, nr, kr, bl);
130
1/2
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
 2616 Buffer imp_packed_rhs(imp_packed_rhs_size);
131
1/2
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
 2616 auto rhs_packed_offset = pack_interface.get_packed_offset(rect_start_row, K, nr, kr, bl);
132
133 // Runs the RHS packing micro-kernel.
134 2616 kai_rhs_pack_nxk_qai4c32p_params params{};
135 2616 params.lhs_zero_point = 1;
136 2616 params.rhs_zero_point = 8;
137
138
5/10
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 654 times.
✓ Branch 3 taken 1962 times.
✓ Branch 4 taken 654 times.
✓ Branch 5 taken 1962 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
 5232 pack_interface.run_pack(
139 2616 1, N, K, nr, kr, sr, bl,
140
3/4
✓ Branch 0 taken 1962 times.
✓ Branch 1 taken 654 times.
✓ Branch 2 taken 1962 times.
✗ Branch 3 not taken.
 2616 reinterpret_cast<const uint8_t*>(s0s1_input ? convert_s0s1_s1s0(rhs_qau4s1s0).data() : rhs_qau4s1s0.data()),
141
2/2
✓ Branch 0 taken 1308 times.
✓ Branch 1 taken 1308 times.
 2616 rhs_zp.data(), has_bias ? biases.data() : nullptr, rhs_scales.data(), imp_packed_rhs.data(), 0, &params);
142
143 2616 return {std::move(imp_packed_rhs), rhs_packed_offset};
144 2616 }
145
146 class MatMulTest_f32_qsi8d32p_qai4c32p : public ::testing::TestWithParam<MatMulTestPortionedParamsWithBias_WithBL> {};
147
148
7/14
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
✓ Branch 2 taken 1 time.
✗ Branch 3 not taken.
✓ Branch 4 taken 1 time.
✗ Branch 5 not taken.
✓ Branch 6 taken 1 time.
✗ Branch 7 not taken.
✓ Branch 8 taken 1 time.
✗ Branch 9 not taken.
✓ Branch 10 taken 1 time.
✗ Branch 11 not taken.
✓ Branch 12 taken 1 time.
✗ Branch 13 not taken.
 16802 TEST_P(MatMulTest_f32_qsi8d32p_qai4c32p, LhsPackedWithSameBlockdepth) {
149 // Verify LHS quant and pack int8 kernel behaves same for int4 and int8 matmul kernels,
150 // when the block-depth is same for different values of kr, sr.
151
152 4797632 const auto& [variant_index, matmul_shape, bl, portion, has_bias] = GetParam();
153 11200 const auto& ukernel_variant = variants_kai_matmul_clamp_f32_qsi8d32p_qai4c32p.at(variant_index);
154
155
2/4
✓ Branch 0 taken 5600 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 5600 times.
✗ Branch 3 not taken.
 5600 if (ukernel_variant.ukernel.fn_is_supported && !ukernel_variant.ukernel.fn_is_supported()) {
156 ✗ GTEST_SKIP() << "Unsupported CPU feature";
157 }
158
159 5600 const std::uint32_t seed = 0;
160
161 11200 const size_t M = matmul_shape.m;
162 11200 const size_t N = matmul_shape.n;
163 11200 const size_t K = matmul_shape.k;
164
165
4/4
✓ Branch 0 taken 1456 times.
✓ Branch 1 taken 4144 times.
✓ Branch 2 taken 1456 times.
✓ Branch 3 taken 4144 times.
 11200 if (K % bl != 0) {
166
3/6
✓ Branch 0 taken 1456 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1456 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1456 times.
✗ Branch 5 not taken.
 1456 GTEST_SKIP() << "K must be a multiple of bl";
167 }
168
169 4144 const auto mr = ukernel_variant.ukernel.interface.get_mr();
170 4144 const auto nr = ukernel_variant.ukernel.interface.get_nr();
171 4144 const auto kr = ukernel_variant.ukernel.interface.get_kr();
172 4144 const auto sr = ukernel_variant.ukernel.interface.get_sr();
173
174 4144 auto m_step = ukernel_variant.ukernel.interface.get_m_step();
175
3/14
✓ Branch 0 taken 4144 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 4144 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✓ Branch 13 taken 4144 times.
 4144 ASSERT_TRUE(m_step % mr == 0);
176
177 4144 auto n_step = ukernel_variant.ukernel.interface.get_n_step();
178
3/14
✓ Branch 0 taken 4144 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 4144 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✓ Branch 13 taken 4144 times.
 4144 ASSERT_TRUE(n_step % nr == 0);
179
180 8288 const auto rect = portion.compute_portion(M, N, m_step, n_step);
181
182 // Generates input data.
183 4144 const auto ref_lhs = fill_random<float>(M * K, seed + 0);
184
185 // Runs the reference implementation.
186 // * Quantizes the LHS matrix using 8-bit symmetric quantization.
187 8288 const auto [ref_lhs_qvalues, ref_lhs_scales] =
188
3/6
✓ Branch 0 taken 4144 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 4144 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 4144 times.
✗ Branch 5 not taken.
 4144 quantize_symmetric_per_block_dynamic<float, int8_t, float>(ref_lhs.data(), M, K, bl);
189
190 // Runs the LHS packing micro-kernel.
191
1/2
✓ Branch 0 taken 4144 times.
✗ Branch 1 not taken.
 4144 const auto lhs_start_row = rect.start_row();
192 4144 auto lhs_stride = K * sizeof(float);
193
194
1/2
✓ Branch 0 taken 4144 times.
✗ Branch 1 not taken.
 12432 auto [imp_packed_lhs, lhs_packed_offset] = pack_lhs_qsi8d32p(
195
2/4
✓ Branch 0 taken 4144 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 4144 times.
✗ Branch 3 not taken.
 8288 ukernel_variant.lhs_pack_interface, M, K, bl, mr, kr, sr, ref_lhs, lhs_stride, lhs_start_row, rect.height());
196
197 4144 const size_t kr_qsi8 = kr / sr;
198 4144 const size_t sr_qsi8 = 1;
199
200
1/2
✓ Branch 0 taken 4144 times.
✗ Branch 1 not taken.
 8288 auto [imp_packed_lhs_qsi8, lhs_qsi8_packed_offset] = pack_lhs_qsi8d32p(
201 8288 ukernel_variant.lhs_pack_interface, M, K, bl, mr, kr_qsi8, sr_qsi8, ref_lhs, lhs_stride, lhs_start_row,
202
1/2
✓ Branch 0 taken 4144 times.
✗ Branch 1 not taken.
 4144 rect.height());
203
204
5/18
✓ Branch 0 taken 4144 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 4144 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 4144 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 4144 times.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
✗ Branch 16 not taken.
✓ Branch 17 taken 4144 times.
 8288 ASSERT_EQ(lhs_qsi8_packed_offset, lhs_packed_offset);
205
206
2/4
✓ Branch 0 taken 4144 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 4144 times.
✗ Branch 3 not taken.
 8288 auto* imp_packed_lhs_ptr = reinterpret_cast<const uint8_t*>(imp_packed_lhs.data());
207
2/4
✓ Branch 0 taken 4144 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 4144 times.
✗ Branch 3 not taken.
 8288 auto* imp_packed_lhs_qsi8_ptr = reinterpret_cast<const uint8_t*>(imp_packed_lhs_qsi8.data());
208
5/8
✗ Branch 0 not taken.
✓ Branch 1 taken 4751600 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 4751600 times.
✓ Branch 4 taken 4144 times.
✓ Branch 5 taken 4747456 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 4144 times.
 4751600 for (size_t i = 0; i < ukernel_variant.lhs_pack_interface.packed_size(M, K, bl, mr, kr, sr); i++) {
209
4/16
✓ Branch 0 taken 4747456 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 4747456 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 4747456 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✓ Branch 15 taken 4747456 times.
 4747456 ASSERT_EQ(imp_packed_lhs_ptr[i], imp_packed_lhs_qsi8_ptr[i]);
210 4747456 }
211 5600 }
212
213
7/14
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
✓ Branch 2 taken 1 time.
✗ Branch 3 not taken.
✓ Branch 4 taken 1 time.
✗ Branch 5 not taken.
✓ Branch 6 taken 1 time.
✗ Branch 7 not taken.
✓ Branch 8 taken 1 time.
✗ Branch 9 not taken.
✓ Branch 10 taken 1 time.
✗ Branch 11 not taken.
✓ Branch 12 taken 1 time.
✗ Branch 13 not taken.
 16802 TEST_P(MatMulTest_f32_qsi8d32p_qai4c32p, EndToEnd) {
214 67848 const auto& [variant_index, matmul_shape, bl, portion, has_bias] = GetParam();
215 11200 const auto& ukernel_variant = variants_kai_matmul_clamp_f32_qsi8d32p_qai4c32p.at(variant_index);
216
217
2/4
✓ Branch 0 taken 5600 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 5600 times.
✗ Branch 3 not taken.
 5600 if (ukernel_variant.ukernel.fn_is_supported && !ukernel_variant.ukernel.fn_is_supported()) {
218 ✗ GTEST_SKIP() << "Unsupported CPU feature";
219 }
220
221 5600 const std::uint32_t seed = 0;
222
223 11200 const size_t M = matmul_shape.m;
224 11200 const size_t N = matmul_shape.n;
225 11200 const size_t K = matmul_shape.k;
226
227
4/4
✓ Branch 0 taken 1456 times.
✓ Branch 1 taken 4144 times.
✓ Branch 2 taken 1456 times.
✓ Branch 3 taken 4144 times.
 11200 if (K % bl != 0) {
228
3/6
✓ Branch 0 taken 1456 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1456 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1456 times.
✗ Branch 5 not taken.
 1456 GTEST_SKIP() << "K must be a multiple of bl";
229 }
230
231 4144 const auto mr = ukernel_variant.ukernel.interface.get_mr();
232 4144 const auto nr = ukernel_variant.ukernel.interface.get_nr();
233 4144 const auto kr = ukernel_variant.ukernel.interface.get_kr();
234 4144 const auto sr = ukernel_variant.ukernel.interface.get_sr();
235
236
4/4
✓ Branch 0 taken 2072 times.
✓ Branch 1 taken 2072 times.
✓ Branch 2 taken 784 times.
✓ Branch 3 taken 1288 times.
 4144 if (mr == 1 && M > 1) {
237
3/6
✓ Branch 0 taken 1288 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1288 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1288 times.
✗ Branch 5 not taken.
 1288 GTEST_SKIP() << "Kernel does not support M != 1";
238 }
239
240 2856 auto m_step = ukernel_variant.ukernel.interface.get_m_step();
241
3/14
✓ Branch 0 taken 2856 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2856 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✓ Branch 13 taken 2856 times.
 2856 ASSERT_TRUE(m_step % mr == 0);
242
243 2856 auto n_step = ukernel_variant.ukernel.interface.get_n_step();
244
3/14
✓ Branch 0 taken 2856 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2856 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✓ Branch 13 taken 2856 times.
 2856 ASSERT_TRUE(n_step % nr == 0);
245
246 5712 const auto rect = portion.compute_portion(M, N, m_step, n_step);
247
4/4
✓ Branch 0 taken 2632 times.
✓ Branch 1 taken 224 times.
✓ Branch 2 taken 16 times.
✓ Branch 3 taken 2616 times.
 2856 if (rect.height() == 0 || rect.width() == 0) {
248
9/18
✓ Branch 0 taken 240 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 240 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 240 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 240 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 240 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 240 times.
✗ Branch 11 not taken.
✓ Branch 12 taken 240 times.
✗ Branch 13 not taken.
✓ Branch 14 taken 240 times.
✗ Branch 15 not taken.
✓ Branch 16 taken 240 times.
✗ Branch 17 not taken.
 240 GTEST_SKIP() << "Empty dimension of matrix(" << rect.width() << "," << rect.height() << ")";
249 }
250
251 // Generates input data.
252 2616 const auto ref_lhs = fill_random<float>(M * K, seed + 0);
253
1/2
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
 2616 const auto ref_rhs = fill_random<float>(N * K, seed + 1);
254 2616 Buffer ref_biases;
255
256
2/2
✓ Branch 0 taken 1308 times.
✓ Branch 1 taken 1308 times.
 2616 if (has_bias) {
257
1/2
✓ Branch 0 taken 1308 times.
✗ Branch 1 not taken.
 1308 ref_biases = fill_random<float>(N, seed + 2);
258 1308 }
259 // Runs the reference implementation.
260 // * Quantizes the LHS matrix using 8-bit symmetric quantization.
261 // * Quantizes the RHS matrix using 8-bit asymmetric quantization.
262 // * Performs GEMM.
263 7848 const auto [ref_lhs_qvalues, ref_lhs_scales] =
264
3/6
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2616 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 2616 times.
✗ Branch 5 not taken.
 2616 quantize_symmetric_per_block_dynamic<float, int8_t, float>(ref_lhs.data(), M, K, bl);
265 275664 const auto [ref_rhs_qai4, ref_rhs_scales, ref_rhs_zero_points] =
266
3/6
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2616 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 2616 times.
✗ Branch 5 not taken.
 2616 quantize_asymmetric_per_block_dynamic<float, Int4, float, int32_t>(ref_rhs.data(), N, K, bl);
267
268 2616 const auto ref_dst_no_clamp =
269
1/2
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
 2616 matmul_nt_t_quantized<int8_t, float, int32_t, Int4, float, int32_t, float, float, int32_t, float>(
270
5/10
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2616 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 2616 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 2616 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 2616 times.
✗ Branch 9 not taken.
 5232 M, N, K, ref_lhs_qvalues.data(), ref_lhs_scales.data(), nullptr, 1, bl, ref_rhs_qai4.data(),
271
7/10
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2616 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1308 times.
✓ Branch 5 taken 1308 times.
✓ Branch 6 taken 1308 times.
✓ Branch 7 taken 1308 times.
✓ Branch 8 taken 1308 times.
✗ Branch 9 not taken.
 2616 ref_rhs_scales.data(), ref_rhs_zero_points.data(), 1, bl, has_bias ? ref_biases.data() : nullptr, nullptr,
272 nullptr, 1);
273
274 // Clamps the reference output.
275 2616 const auto clamp_ratio = 0.8F;
276
2/4
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2616 times.
✗ Branch 3 not taken.
 7848 const auto [clamp_min, clamp_max] = find_clamp_range<float>(ref_dst_no_clamp.data(), M * N, clamp_ratio);
277
4/8
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2616 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 2616 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 2616 times.
✗ Branch 7 not taken.
 2616 const auto ref_dst = clamp<float>(ref_dst_no_clamp.data(), M * N, clamp_min, clamp_max);
278
279 // Runs the LHS packing micro-kernel.
280
1/2
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
 2616 const auto lhs_start_row = rect.start_row();
281
1/2
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
 5232 auto [imp_packed_lhs, lhs_packed_offset] = pack_lhs_qsi8d32p(
282 5232 ukernel_variant.lhs_pack_interface, M, K, bl, mr, kr, sr, ref_lhs, K * sizeof(float), lhs_start_row,
283
1/2
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
 2616 rect.height());
284
2/4
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2616 times.
✗ Branch 3 not taken.
 5232 auto lhs_matmul_offset = ukernel_variant.ukernel.interface.get_lhs_packed_offset(lhs_start_row, K, bl);
285
286
5/18
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2616 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 2616 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 2616 times.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
✗ Branch 16 not taken.
✓ Branch 17 taken 2616 times.
 5232 ASSERT_EQ(lhs_packed_offset, lhs_matmul_offset);
287
288 // Prepare the offsets as the RHS packing micro-kernel expects the scaled zero-points in float.
289
2/4
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2616 times.
✗ Branch 3 not taken.
 5232 const size_t num_blocks_per_row = round_up_division(K, bl);
290 2616 const size_t ref_zp_size = N * num_blocks_per_row;
291 2616 const size_t ref_zp_size_in_bytes = ref_zp_size * sizeof(float);
292
1/2
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
 2616 Buffer ref_rhs_zp_f32(ref_zp_size_in_bytes);
293
2/2
✓ Branch 0 taken 262584 times.
✓ Branch 1 taken 2616 times.
 265200 for (size_t i = 0; i < ref_zp_size; ++i) {
294
1/2
✓ Branch 0 taken 262584 times.
✗ Branch 1 not taken.
 262584 reinterpret_cast<float*>(ref_rhs_zp_f32.data())[i] =
295
1/2
✓ Branch 0 taken 262584 times.
✗ Branch 1 not taken.
 262584 -reinterpret_cast<const int32_t*>(ref_rhs_zero_points.data())[i] *
296
1/2
✓ Branch 0 taken 262584 times.
✗ Branch 1 not taken.
 262584 reinterpret_cast<const float*>(ref_rhs_scales.data())[i];
297 262584 }
298
299
1/2
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
 2616 const auto rhs_start_row = rect.start_col();
300
2/4
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2616 times.
✗ Branch 3 not taken.
 7848 auto [imp_packed_rhs, rhs_packed_offset] = pack_rhs_qai4c32p(
301 7848 ukernel_variant.rhs_pack_interface, N, K, bl, nr, kr, sr, ref_rhs_qai4, has_bias, ref_biases, ref_rhs_scales,
302 2616 ref_rhs_zp_f32, ukernel_variant.rhs_s0s1_input, rhs_start_row);
303
304
2/4
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2616 times.
✗ Branch 3 not taken.
 5232 auto rhs_matmul_offset = ukernel_variant.ukernel.interface.get_rhs_packed_offset(rhs_start_row, K, bl);
305
5/18
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2616 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 2616 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 2616 times.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
✗ Branch 16 not taken.
✓ Branch 17 taken 2616 times.
 5232 ASSERT_EQ(rhs_packed_offset, rhs_matmul_offset);
306
307 2616 const auto dst_stride_row = N * sizeof(float);
308 2616 const auto dst_stride_col = sizeof(float);
309 5232 const auto dst_offset =
310
3/6
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2616 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 2616 times.
✗ Branch 5 not taken.
 2616 ukernel_variant.ukernel.interface.get_dst_offset(rect.start_row(), rect.start_col(), dst_stride_row);
311
2/4
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2616 times.
✗ Branch 3 not taken.
 2616 const auto ref_dst_offset = rect.start_row() * dst_stride_row + rect.start_col() * dst_stride_col;
312
4/16
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2616 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 2616 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✓ Branch 15 taken 2616 times.
 2616 ASSERT_EQ(dst_offset, ref_dst_offset);
313
314 // Runs the GEMM micro-kernel.
315
1/2
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
 2616 const auto imp_dst_size = ukernel_variant.ukernel.interface.get_dst_size(M, N);
316
5/18
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2616 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 2616 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 2616 times.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
✗ Branch 16 not taken.
✓ Branch 17 taken 2616 times.
 2616 ASSERT_EQ(imp_dst_size, ref_dst.size());
317
1/2
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
 2616 Buffer imp_dst(imp_dst_size);
318
1/2
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
 5232 ukernel_variant.ukernel.interface.run_matmul(
319
5/10
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2616 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 2616 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 2616 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 2616 times.
✗ Branch 9 not taken.
 2616 rect.height(), rect.width(), K, bl, imp_packed_lhs.data() + lhs_matmul_offset,
320
2/4
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2616 times.
✗ Branch 3 not taken.
 2616 imp_packed_rhs.data() + rhs_matmul_offset, reinterpret_cast<float*>(imp_dst.data() + dst_offset),
321 7848 dst_stride_row, dst_stride_col, clamp_min, clamp_max);
322
323 // Compares the output of the micro-kernels against the output of the reference implementation for the portion
324 // tested.
325
1/2
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
 2616 DefaultMismatchHandler handler(0, 0.1, 0, 0.05);
326 2616 DataFormat dst_format = DataFormat(DataType::FP32);
327
3/6
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2616 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 2616 times.
✗ Branch 5 not taken.
 2616 const auto success = compare(imp_dst.data(), ref_dst.data(), dst_format, M, N, rect, handler);
328
4/16
✓ Branch 0 taken 2616 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2616 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 2616 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✓ Branch 15 taken 2616 times.
 2616 ASSERT_TRUE(success);
329 5600 }
330
37/96
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
✓ Branch 2 taken 1 time.
✗ Branch 3 not taken.
✓ Branch 4 taken 1 time.
✗ Branch 5 not taken.
✓ Branch 6 taken 1 time.
✗ Branch 7 not taken.
✓ Branch 8 taken 1 time.
✗ Branch 9 not taken.
✓ Branch 10 taken 2 times.
✗ Branch 11 not taken.
✓ Branch 12 taken 2 times.
✗ Branch 13 not taken.
✓ Branch 14 taken 2 times.
✗ Branch 15 not taken.
✓ Branch 16 taken 2 times.
✗ Branch 17 not taken.
✓ Branch 18 taken 2 times.
✗ Branch 19 not taken.
✓ Branch 20 taken 2 times.
✗ Branch 21 not taken.
✓ Branch 22 taken 2 times.
✗ Branch 23 not taken.
✓ Branch 24 taken 2 times.
✗ Branch 25 not taken.
✓ Branch 26 taken 2 times.
✗ Branch 27 not taken.
✓ Branch 28 taken 2 times.
✗ Branch 29 not taken.
✓ Branch 30 taken 2 times.
✗ Branch 31 not taken.
✓ Branch 32 taken 2 times.
✗ Branch 33 not taken.
✓ Branch 34 taken 2 times.
✗ Branch 35 not taken.
✓ Branch 36 taken 11200 times.
✗ Branch 37 not taken.
✗ Branch 38 not taken.
✗ Branch 39 not taken.
✗ Branch 40 not taken.
✗ Branch 41 not taken.
✗ Branch 42 not taken.
✗ Branch 43 not taken.
✗ Branch 44 not taken.
✗ Branch 45 not taken.
✗ Branch 46 not taken.
✗ Branch 47 not taken.
✗ Branch 48 not taken.
✗ Branch 49 not taken.
✗ Branch 50 not taken.
✗ Branch 51 not taken.
✗ Branch 52 not taken.
✗ Branch 53 not taken.
✗ Branch 54 not taken.
✗ Branch 55 not taken.
✗ Branch 56 not taken.
✗ Branch 57 not taken.
✗ Branch 58 not taken.
✗ Branch 59 not taken.
✗ Branch 60 not taken.
✗ Branch 61 not taken.
✗ Branch 62 not taken.
✗ Branch 63 not taken.
✓ Branch 64 taken 11200 times.
✗ Branch 65 not taken.
✓ Branch 66 taken 11200 times.
✗ Branch 67 not taken.
✓ Branch 68 taken 11200 times.
✗ Branch 69 not taken.
✓ Branch 70 taken 11200 times.
✗ Branch 71 not taken.
✓ Branch 72 taken 11200 times.
✗ Branch 73 not taken.
✓ Branch 74 taken 11200 times.
✗ Branch 75 not taken.
✓ Branch 76 taken 11200 times.
✗ Branch 77 not taken.
✓ Branch 78 taken 5600 times.
✓ Branch 79 taken 5600 times.
✓ Branch 80 taken 5600 times.
✗ Branch 81 not taken.
✓ Branch 82 taken 5600 times.
✗ Branch 83 not taken.
✓ Branch 84 taken 11200 times.
✗ Branch 85 not taken.
✓ Branch 86 taken 8400 times.
✓ Branch 87 taken 2800 times.
✓ Branch 88 taken 8400 times.
✗ Branch 89 not taken.
✓ Branch 90 taken 2800 times.
✗ Branch 91 not taken.
✓ Branch 92 taken 11200 times.
✗ Branch 93 not taken.
✓ Branch 94 taken 11200 times.
✗ Branch 95 not taken.
 30803 INSTANTIATE_TEST_SUITE_P(
331 MatMul, MatMulTest_f32_qsi8d32p_qai4c32p,
332 testing::Combine(
333 testing::Range<size_t>(0, variants_kai_matmul_clamp_f32_qsi8d32p_qai4c32p.size()),
334 testing::Values(
335 MatMulShape{1, 64, 32}, //
336 MatMulShape{1, 63, 32}, //
337 MatMulShape{1, 65, 32}, //
338 MatMulShape{1, 64, 64}, //
339 MatMulShape{1, 64, 128}, //
340 MatMulShape{1, 128, 32}, //
341 MatMulShape{1, 128, 128}, //
342 MatMulShape{1, 2, 32}, //
343 MatMulShape{1, 3, 32}, //
344 MatMulShape{1, 4, 32}, //
345 MatMulShape{1, 5, 32}, //
346 MatMulShape{3, 3, 32}, //
347 MatMulShape{4, 4, 32}, //
348 MatMulShape{5, 5, 32}, //
349 MatMulShape{32, 128, 32}, //
350 MatMulShape{15, 64, 64}, //
351 MatMulShape{17, 64, 64}, //
352 MatMulShape{16, 63, 64}, //
353 MatMulShape{16, 64, 64}, //
354 MatMulShape{16, 65, 64}, //
355 MatMulShape{32, 64, 64}, //
356 MatMulShape{16, 32, 64}, //
357 MatMulShape{8, 32, 64}, //
358 MatMulShape{15, 32, 32}, //
359 MatMulShape{77, 99, 64}),
360 testing::Values(32, 64),
361 testing::Values(
362 MatrixPortion(0, 0, 1, 1), // Full matrix.
363 MatrixPortion(0, 0, 1, 0.25), // Leftmost portion.
364 MatrixPortion(0, 0.75, 1, 1), // Rightmost portion.
365 MatrixPortion(0, 0.5, 1, 0.8), // Somewhere Middle
366 MatrixPortion(0.75, 0.75, 1, 1), // Bottom-right corner.
367 MatrixPortion(0.75, 0, 1, 1), // Partial rows
368 MatrixPortion(0.4, 0.5, 0.6, 0.8) // Somewhere Middle
369 ),
370 testing::Bool()),
371 [](const auto& info) {
372 const auto variant_idx = std::get<0>(info.param);
373 const std::string name{variants_kai_matmul_clamp_f32_qsi8d32p_qai4c32p.at(variant_idx).ukernel.name};
374 const auto shape = std::get<MatMulShape>(info.param);
375 const auto bl = std::get<2>(info.param);
376 const auto portion = std::get<3>(info.param);
377 const auto has_bias = std::get<4>(info.param);
378
379 std::ostringstream sstream;
380 sstream << name << "__";
381 PrintTo(shape, &sstream);
382 sstream << "__BL_" << bl << "_";
383 if (has_bias) {
384 sstream << "_withBias_";
385 } else {
386 sstream << "_noBias_";
387 }
388 if (variants_kai_matmul_clamp_f32_qsi8d32p_qai4c32p.at(variant_idx).rhs_s0s1_input) {
389 sstream << "_RHS_s0s1__";
390 } else {
391 sstream << "_RHS_s1s0__";
392 }
393 PrintTo(portion, &sstream);
394
395 return sstream.str();
396 });
397
398 } // namespace kai::test
399