KleidiAI Coverage Report

Directory: ./
Coverage: low: ≥ 0% medium: ≥ 75.0% high: ≥ 90.0%
Coverage Exec / Excl / Total
Lines: 98.1% 52 / 5 / 58
Functions: 100.0% 14 / 0 / 14
Branches: 50.0% 1 / 10 / 12
kai/ukernels/matmul/matmul_clamp_bf16_qai8dxp_qsi4cxp/kai_matmul_clamp_bf16_qai8dxp4x8_qsi4cxp8x8_8x8_neon_i8mm.c
Line Branch Exec Source
1 //
2 // SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its affiliates <open-source-office@arm.com>
3 //
4 // SPDX-License-Identifier: Apache-2.0
5 //
6 #if ( \
7 !defined(__aarch64__) || !defined(__ARM_FEATURE_MATMUL_INT8) || !defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC)) && \
8 !defined(_M_ARM64)
9 #error "I8mm extension and bf16 vector arithmetic required to compile this micro-kernel"
10 #else // Architectural features check.
11
12 #include "kai_matmul_clamp_bf16_qai8dxp4x8_qsi4cxp8x8_8x8_neon_i8mm.h"
13
14 #include <stddef.h>
15 #include <stdint.h>
16
17 #include "kai/kai_common.h"
18
19 typedef struct {
20 void* dst;
21 const void* lhs_packed;
22 const void* rhs_packed;
23 const float* clamp_vals;
24 size_t dst_stride_row;
25 size_t m;
26 size_t n;
27 size_t num_blocks;
28 } KernelArgs;
29
30 void kai_kernel_matmul_clamp_bf16_qai8dxp4x8_qsi4cxp8x8_8x8_neon_i8mm(KernelArgs* args_ptr);
31
32 // Compute args
33 static const size_t kai_m_step = 8;
34 static const size_t kai_n_step = 8;
35 // Packing args
36 static const size_t kai_mr = 4;
37 static const size_t kai_nr = 8;
38 static const size_t kai_kr = 16;
39 static const size_t kai_sr = 2;
40 // LHS format args (num. bytes per value, multiplier, zero_point (if asymmetric))
41 static const size_t kai_num_bytes_qvalue_lhs = 1;
42 static const size_t kai_num_bytes_multiplier_lhs = 4;
43 static const size_t kai_num_bytes_zp_lhs = 4;
44 // RHS format args (num. bytes per value, multiplier, zero_point (if asymmetric), and reduction sum (if LHS is
45 // asymmetric))
46 static const size_t kai_num_bytes_recip_qvalue_rhs = 2;
47 static const size_t kai_num_bytes_multiplier_rhs = 4;
48 static const size_t kai_num_bytes_rsum_rhs = 4;
49 // DST format args
50 static const size_t kai_num_bytes_dst_value = 2;
51 // Extra args
52 static const size_t kai_num_bytes_bias = 4;
53 static const size_t kai_k_multiple_of = 32;
54
55 static const size_t kai_bl = 32;
56
57 7062 inline static size_t kai_get_k_roundedup(size_t k) {
58 7062 return kai_roundup(k, kai_k_multiple_of);
59 }
60
61 2352 inline static size_t kai_get_lhs_packed_stride(size_t k) {
62 2352 const size_t k_internal = kai_get_k_roundedup(k);
63 2352 size_t lhs_packed_stride = kai_mr * ((k_internal * kai_num_bytes_qvalue_lhs) + kai_num_bytes_multiplier_lhs);
64 // Since the LHS matrix is asymmetric with per-row quantization, we must include the
65 // the number of bytes to hold the zero point value
66 2352 lhs_packed_stride += kai_mr * kai_num_bytes_zp_lhs;
67
68 4704 return lhs_packed_stride;
69 2352 }
70
71 2352 inline static size_t kai_get_rhs_packed_stride(size_t k) {
72 2352 const size_t k_internal = kai_get_k_roundedup(k);
73 2352 size_t rhs_packed_stride = kai_nr * (k_internal / kai_num_bytes_recip_qvalue_rhs);
74
75 2352 rhs_packed_stride += kai_nr * kai_num_bytes_multiplier_rhs;
76 // Since the LHS matrix is quantized asymmetric with per-row quantization, we also include
77 // the number of bytes for the reduction sum
78 2352 rhs_packed_stride += kai_nr * kai_num_bytes_rsum_rhs;
79 // Since the bias is packed with the RHS matrix, the stride is adjusted with the number of bytes of the bias
80 2352 rhs_packed_stride += kai_nr * kai_num_bytes_bias;
81
82 4704 return rhs_packed_stride;
83 2352 }
84
85 2352 size_t kai_get_m_step_matmul_clamp_bf16_qai8dxp4x8_qsi4cxp8x8_8x8_neon_i8mm(void) {
86 2352 return kai_m_step;
87 }
88
89 2352 size_t kai_get_n_step_matmul_clamp_bf16_qai8dxp4x8_qsi4cxp8x8_8x8_neon_i8mm(void) {
90 2352 return kai_n_step;
91 }
92
93 2352 size_t kai_get_mr_matmul_clamp_bf16_qai8dxp4x8_qsi4cxp8x8_8x8_neon_i8mm(void) {
94 2352 return kai_mr;
95 }
96
97 2352 size_t kai_get_nr_matmul_clamp_bf16_qai8dxp4x8_qsi4cxp8x8_8x8_neon_i8mm(void) {
98 2352 return kai_nr;
99 }
100
101 2352 size_t kai_get_kr_matmul_clamp_bf16_qai8dxp4x8_qsi4cxp8x8_8x8_neon_i8mm(void) {
102 2352 return kai_kr;
103 }
104
105 2352 size_t kai_get_sr_matmul_clamp_bf16_qai8dxp4x8_qsi4cxp8x8_8x8_neon_i8mm(void) {
106 2352 return kai_sr;
107 }
108
109 2352 size_t kai_get_lhs_packed_offset_matmul_clamp_bf16_qai8dxp4x8_qsi4cxp8x8_8x8_neon_i8mm(size_t m_idx, size_t k) {
110 − KAI_ASSUME((m_idx % kai_m_step) == 0);
111
112 2352 return (m_idx / kai_mr) * kai_get_lhs_packed_stride(k);
113 }
114
115 2352 size_t kai_get_rhs_packed_offset_matmul_clamp_bf16_qai8dxp4x8_qsi4cxp8x8_8x8_neon_i8mm(size_t n_idx, size_t k) {
116 − KAI_ASSUME((n_idx % kai_n_step) == 0);
117
118 2352 return (n_idx / kai_nr) * kai_get_rhs_packed_stride(k);
119 }
120
121 2352 size_t kai_get_dst_offset_matmul_clamp_bf16_qai8dxp4x8_qsi4cxp8x8_8x8_neon_i8mm(
122 size_t m_idx, size_t n_idx, size_t dst_stride) {
123 − KAI_ASSUME((m_idx % kai_m_step) == 0);
124 − KAI_ASSUME((n_idx % kai_n_step) == 0);
125
126 2352 return (n_idx * kai_num_bytes_dst_value) + m_idx * dst_stride;
127 }
128
129 2352 size_t kai_get_dst_size_matmul_clamp_bf16_qai8dxp4x8_qsi4cxp8x8_8x8_neon_i8mm(size_t m, size_t n) {
130 2352 return m * n * kai_num_bytes_dst_value;
131 }
132
133 2358 void kai_run_matmul_clamp_bf16_qai8dxp4x8_qsi4cxp8x8_8x8_neon_i8mm(
134 size_t m, //
135 size_t n, //
136 size_t k, //
137 const void* restrict lhs_packed, //
138 const void* restrict rhs_packed, //
139 void* restrict dst, // NOLINT(readability-non-const-parameter)
140 size_t dst_stride_row, //
141 size_t dst_stride_col, //
142 float scalar_min, //
143 float scalar_max) {
144 − KAI_ASSUME(dst_stride_col == sizeof(uint16_t));
145
1/2
✓ Branch 0 taken 2358 times.
✗ Branch 1 not taken.
 2358 if (m == 0) {
146 ✗ return;
147 }
148 2358 const size_t k_internal = kai_get_k_roundedup(k);
149 2358 size_t num_blocks = k_internal / kai_bl;
150 2358 const float clamp_vals[2] = {scalar_min, scalar_max};
151
152 2358 KernelArgs args;
153
154 2358 args.dst = dst;
155 2358 args.lhs_packed = lhs_packed;
156 2358 args.rhs_packed = rhs_packed;
157 2358 args.clamp_vals = clamp_vals;
158 2358 args.dst_stride_row = dst_stride_row;
159 2358 args.m = m;
160 2358 args.n = n;
161 2358 args.num_blocks = num_blocks;
162
163 2358 kai_kernel_matmul_clamp_bf16_qai8dxp4x8_qsi4cxp8x8_8x8_neon_i8mm(&args);
164 2358 }
165
166 #endif // Architectural features check.
167