KleidiAI Coverage Report

Directory: ./
File: kai/ukernels/matmul/matmul_clamp_bf16_qai8dxp_qsi4c32p/kai_matmul_clamp_bf16_qai8dxp4x8_qsi4c32p4x8_16x4_neon_i8mm.c
Date: 2025-10-20 13:18:31
Coverage Exec Excl Total
Lines: 98.3% 59 11 71
Functions: 100.0% 16 0 16
Branches: 50.0% 1 22 24
Line Branch Exec Source
1 //
2 // SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its affiliates <open-source-office@arm.com>
3 //
4 // SPDX-License-Identifier: Apache-2.0
5 //
6 #if ( \
7 !defined(__aarch64__) || !defined(__ARM_FEATURE_MATMUL_INT8) || !defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC)) && \
8 !defined(_M_ARM64)
9 #error "I8mm extension and bf16 vector arithmetic required to compile this micro-kernel"
10 #else // Architectural features check.
11
12 #include "kai_matmul_clamp_bf16_qai8dxp4x8_qsi4c32p4x8_16x4_neon_i8mm.h"
13
14 #include <stddef.h>
15 #include <stdint.h>
16
17 #include "kai/kai_common.h"
18
19 typedef struct {
20 void* dst;
21 const void* lhs_packed;
22 const void* rhs_packed;
23 const float* clamp_vals;
24 size_t dst_stride_row;
25 size_t m;
26 size_t n;
27 size_t num_blocks;
28 size_t num_subblocks;
29 } KernelArgs;
30
31 void kai_kernel_matmul_clamp_bf16_qai8dxp4x8_qsi4c32p4x8_16x4_neon_i8mm(KernelArgs* args_ptr);
32
33 // Compute args
34 static const size_t kai_m_step = 16;
35 static const size_t kai_n_step = 4;
36 // Packing args
37 static const size_t kai_mr = 4;
38 static const size_t kai_nr = 4;
39 static const size_t kai_kr = 16;
40 static const size_t kai_sr = 2;
41 // LHS format args (num. bytes per value, multiplier, zero_point (if asymmetric))
42 static const size_t kai_num_bytes_qvalue_lhs = 1;
43 static const size_t kai_num_bytes_multiplier_lhs = 4;
44 static const size_t kai_num_bytes_zp_lhs = 4;
45 // RHS format args (num. bytes per value, multiplier, zero_point (if asymmetric), and reduction sum (if LHS is
46 // asymmetric))
47 static const size_t kai_num_bytes_recip_qvalue_rhs = 2;
48 static const size_t kai_num_bytes_multiplier_rhs = 2;
49 static const size_t kai_num_bytes_rsum_rhs = 4;
50 // DST format args
51 static const size_t kai_num_bytes_dst_value = 2;
52 // Extra args
53 static const size_t kai_num_bytes_bias = 4;
54 static const size_t kai_k_multiple_of = 32;
55 static const size_t kai_bl = 32;
56
57 108 inline static size_t kai_get_k_roundedup(size_t k) {
58 108 return kai_roundup(k, kai_k_multiple_of);
59 }
60
61 108 inline static size_t kai_get_num_bytes_per_block_rhs(size_t bl) {
62 − KAI_ASSUME((bl % kai_bl) == 0);
63 108 size_t num_bytes_per_block_rhs = (bl / kai_num_bytes_recip_qvalue_rhs) + kai_num_bytes_multiplier_rhs;
64 216 return num_bytes_per_block_rhs;
65 108 }
66
67 271 inline static size_t kai_get_num_blocks_per_row(size_t k, size_t bl) {
68 − KAI_ASSUME((bl % kai_bl) == 0);
69
70 271 return kai_roundup(k, bl) / bl;
71 }
72
73 108 inline static size_t kai_get_lhs_packed_stride(size_t k) {
74 108 const size_t k_internal = kai_get_k_roundedup(k);
75 108 size_t lhs_packed_stride = kai_mr * ((k_internal * kai_num_bytes_qvalue_lhs) + kai_num_bytes_multiplier_lhs);
76 // Since the LHS matrix is asymmetric with per-row quantization, we must include the
77 // the number of bytes to hold the zero point value
78 108 lhs_packed_stride += kai_mr * kai_num_bytes_zp_lhs;
79
80 216 return lhs_packed_stride;
81 108 }
82
83 108 inline static size_t kai_get_rhs_packed_stride(size_t k, size_t bl) {
84 − KAI_ASSUME((bl % kai_bl) == 0);
85
86 108 const size_t num_blocks_per_row = kai_get_num_blocks_per_row(k, bl);
87 108 const size_t num_bytes_per_block = kai_get_num_bytes_per_block_rhs(bl);
88
89 108 size_t rhs_packed_stride = kai_nr * (num_bytes_per_block * num_blocks_per_row);
90 // Since the LHS matrix is quantized asymmetric with per-row quantization, we also include
91 // the number of bytes for the reduction sum
92 108 rhs_packed_stride += kai_nr * kai_num_bytes_rsum_rhs;
93 // Since the bias is packed with the RHS matrix, the stride is adjusted with the number of bytes of the bias
94 108 rhs_packed_stride += kai_nr * kai_num_bytes_bias;
95
96 216 return rhs_packed_stride;
97 108 }
98
99 112 size_t kai_get_m_step_matmul_clamp_bf16_qai8dxp4x8_qsi4c32p4x8_16x4_neon_i8mm(void) {
100 112 return kai_m_step;
101 }
102
103 112 size_t kai_get_n_step_matmul_clamp_bf16_qai8dxp4x8_qsi4c32p4x8_16x4_neon_i8mm(void) {
104 112 return kai_n_step;
105 }
106
107 112 size_t kai_get_mr_matmul_clamp_bf16_qai8dxp4x8_qsi4c32p4x8_16x4_neon_i8mm(void) {
108 112 return kai_mr;
109 }
110
111 112 size_t kai_get_nr_matmul_clamp_bf16_qai8dxp4x8_qsi4c32p4x8_16x4_neon_i8mm(void) {
112 112 return kai_nr;
113 }
114
115 112 size_t kai_get_kr_matmul_clamp_bf16_qai8dxp4x8_qsi4c32p4x8_16x4_neon_i8mm(void) {
116 112 return kai_kr;
117 }
118
119 112 size_t kai_get_sr_matmul_clamp_bf16_qai8dxp4x8_qsi4c32p4x8_16x4_neon_i8mm(void) {
120 112 return kai_sr;
121 }
122
123 108 size_t kai_get_lhs_packed_offset_matmul_clamp_bf16_qai8dxp4x8_qsi4c32p4x8_16x4_neon_i8mm(size_t m_idx, size_t k) {
124 − KAI_ASSUME((m_idx % kai_m_step) == 0);
125
126 108 return (m_idx / kai_mr) * kai_get_lhs_packed_stride(k);
127 }
128
129 108 size_t kai_get_rhs_packed_offset_matmul_clamp_bf16_qai8dxp4x8_qsi4c32p4x8_16x4_neon_i8mm(
130 size_t n_idx, size_t k, size_t bl) {
131 − KAI_ASSUME((k % bl) == 0);
132 − KAI_ASSUME((n_idx % kai_n_step) == 0);
133
134 108 return (n_idx / kai_nr) * kai_get_rhs_packed_stride(k, bl);
135 }
136
137 108 size_t kai_get_dst_offset_matmul_clamp_bf16_qai8dxp4x8_qsi4c32p4x8_16x4_neon_i8mm(
138 size_t m_idx, size_t n_idx, size_t dst_stride) {
139 − KAI_ASSUME((m_idx % kai_m_step) == 0);
140 − KAI_ASSUME((n_idx % kai_n_step) == 0);
141
142 108 return (n_idx * kai_num_bytes_dst_value) + m_idx * dst_stride;
143 }
144
145 108 size_t kai_get_dst_size_matmul_clamp_bf16_qai8dxp4x8_qsi4c32p4x8_16x4_neon_i8mm(size_t m, size_t n) {
146 108 return m * n * kai_num_bytes_dst_value;
147 }
148
149 163 void kai_run_matmul_clamp_bf16_qai8dxp4x8_qsi4c32p4x8_16x4_neon_i8mm(
150 size_t m, //
151 size_t n, //
152 size_t k, //
153 size_t bl, //
154 const void* restrict lhs_packed, //
155 const void* restrict rhs_packed, //
156 void* restrict dst, // NOLINT(readability-non-const-parameter)
157 size_t dst_stride_row, //
158 size_t dst_stride_col, //
159 float scalar_min, //
160 float scalar_max) {
161 − KAI_ASSUME(dst_stride_col == sizeof(uint16_t));
162 − KAI_ASSUME((k % bl) == 0);
163 − KAI_ASSUME((bl % kai_bl) == 0);
164
165
1/2
✓ Branch 0 taken 163 times.
✗ Branch 1 not taken.
 163 if (m == 0) {
166 ✗ return;
167 }
168 163 const size_t num_subblocks = bl / kai_bl;
169 163 const size_t num_blocks = kai_get_num_blocks_per_row(k, bl);
170 163 const float clamp_vals[2] = {scalar_min, scalar_max};
171
172 163 KernelArgs args;
173
174 163 args.dst = dst;
175 163 args.lhs_packed = lhs_packed;
176 163 args.rhs_packed = rhs_packed;
177 163 args.clamp_vals = clamp_vals;
178 163 args.dst_stride_row = dst_stride_row;
179 163 args.m = m;
180 163 args.n = n;
181 163 args.num_blocks = num_blocks;
182 163 args.num_subblocks = num_subblocks;
183
184 163 kai_kernel_matmul_clamp_bf16_qai8dxp4x8_qsi4c32p4x8_16x4_neon_i8mm(&args);
185 163 }
186
187 #endif // Architectural features check.
188