KleidiAI Coverage Report

Directory: ./
Coverage: low: ≥ 0% medium: ≥ 75.0% high: ≥ 90.0%
Coverage Exec / Excl / Total
Lines: 97.6% 41 / 19 / 61
Functions: 100.0% 16 / 0 / 16
Branches: 50.0% 1 / 38 / 40
kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod.c
Line Branch Exec Source
1
2 //
3 // SPDX-FileCopyrightText: Copyright 2024-2025 Arm Limited and/or its affiliates <open-source-office@arm.com>
4 //
5 // SPDX-License-Identifier: Apache-2.0
6 //
7 #if !defined(__ARM_FEATURE_DOTPROD)
8 #error "Dotprod extension required to compile this micro-kernel"
9 #else
10 #include "kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod.h"
11
12 #include <arm_neon.h>
13 #include <stddef.h>
14 #include <stdint.h>
15
16 #include "kai/kai_common.h"
17
18 static const size_t kai_m_step = 1;
19 static const size_t kai_n_step = 4;
20 static const size_t kai_mr = 1;
21 static const size_t kai_nr = 4;
22 static const size_t kai_kr = 16;
23 static const size_t kai_sr = 2;
24 static const size_t kai_bl = 32;
25 static const size_t kai_num_bytes_multiplier = sizeof(uint16_t);
26
27 256 inline static size_t kai_num_bytes_per_block_lhs(void) {
28 256 return kai_bl * sizeof(int8_t) + kai_num_bytes_multiplier;
29 }
30
31 256 inline static size_t kai_num_bytes_per_block_rhs(void) {
32 256 return (kai_bl / 2) * sizeof(int8_t) + kai_num_bytes_multiplier;
33 }
34
35 512 inline static size_t kai_num_blocks_per_row(size_t k) {
36 − KAI_ASSUME((k % kai_bl) == 0);
37 512 return k / kai_bl;
38 }
39
40 256 inline static size_t kai_lhs_packed_stride(size_t k) {
41 256 return kai_mr * kai_num_blocks_per_row(k) * kai_num_bytes_per_block_lhs();
42 }
43
44 256 inline static size_t kai_rhs_packed_stride(size_t k) {
45 − KAI_ASSUME((k % 2) == 0);
46 − KAI_ASSUME((k % kai_kr) == 0);
47 − KAI_ASSUME((k % kai_bl) == 0);
48
49 256 const size_t num_blocks_per_row = kai_num_blocks_per_row(k);
50 256 const size_t num_bytes_per_block = kai_num_bytes_per_block_rhs();
51
52 512 return kai_nr * (num_bytes_per_block * num_blocks_per_row);
53 256 }
54
55 440 size_t kai_get_m_step_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod(void) {
56 440 return kai_m_step;
57 }
58
59 440 size_t kai_get_n_step_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod(void) {
60 440 return kai_n_step;
61 }
62
63 368 size_t kai_get_mr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod(void) {
64 368 return kai_mr;
65 }
66
67 368 size_t kai_get_nr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod(void) {
68 368 return kai_nr;
69 }
70
71 552 size_t kai_get_kr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod(void) {
72 552 return kai_kr;
73 }
74
75 368 size_t kai_get_sr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod(void) {
76 368 return kai_sr;
77 }
78
79 256 size_t kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod(
80 size_t m_idx, size_t k, size_t bl) {
81 − KAI_ASSUME(bl == kai_bl);
82 − KAI_ASSUME((k % 2) == 0);
83 − KAI_ASSUME((k % kai_kr) == 0);
84 − KAI_ASSUME((k % bl) == 0);
85 − KAI_ASSUME((m_idx % kai_m_step) == 0);
86
87 256 return (m_idx / kai_mr) * kai_lhs_packed_stride(k);
88 }
89
90 256 size_t kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod(
91 size_t n_idx, size_t k, size_t bl) {
92 − KAI_ASSUME(bl == kai_bl);
93 − KAI_ASSUME((k % 2) == 0);
94 − KAI_ASSUME((k % kai_kr) == 0);
95 − KAI_ASSUME((k % bl) == 0);
96 − KAI_ASSUME((n_idx % kai_n_step) == 0);
97
98 256 return (n_idx / kai_nr) * kai_rhs_packed_stride(k);
99 }
100
101 72 size_t kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod(
102 size_t m_idx, size_t n_idx, size_t dst_stride) {
103 − KAI_ASSUME((m_idx % kai_m_step) == 0);
104 − KAI_ASSUME((n_idx % kai_n_step) == 0);
105
106 72 return (n_idx * sizeof(float)) + m_idx * dst_stride;
107 }
108
109 72 size_t kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod(size_t m, size_t n) {
110 72 return m * n * sizeof(float);
111 }
112
113 78 void kai_run_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod(
114 size_t m, size_t n, size_t k, size_t bl, const void* lhs_packed, const void* rhs_packed,
115 float* dst, // NOLINT(readability-non-const-parameter)
116 size_t dst_stride_row, size_t dst_stride_col, float scalar_min, float scalar_max) {
117 − KAI_ASSUME(bl == kai_bl);
118 − KAI_ASSUME(k % kai_bl == 0);
119 − KAI_ASSUME(dst_stride_col == sizeof(float));
120
121
1/2
✓ Branch 0 taken 78 times.
✗ Branch 1 not taken.
 78 if (m == 0) {
122 ✗ return;
123 }
124
125 78 const size_t num_blocks = k / kai_bl;
126 78 float clamp_vals[2] = {scalar_min, scalar_max};
127
128 156 __asm__ __volatile__(
129 "mov x26, #0x22\n"
130 "movi v1.16b, #0xf0\n"
131 "mov x25, %x[m]\n"
132 "mul x26, %x[num_blocks], x26\n"
133 "1:" // Row loop
134 "mov x24, %x[rhs_packed]\n"
135 "mov x23, %x[n]\n"
136 "add x22, %x[dst], %x[dst_stride_row]\n"
137 "2:" // Column loop
138 "mov x21, %x[lhs_packed]\n"
139 "movi v0.16b, #0x0\n"
140 "mov x20, %x[num_blocks]\n"
141 "3:" // Block loop
142 "ldr d16, [x24, #0x0]\n"
143 "ld1r { v31.8h }, [x21]\n"
144 "add x24, x24, #0x8\n"
145 "add x21, x21, #0x2\n"
146 "ldr q30, [x24, #0x0]\n"
147 "ldr q29, [x24, #0x10]\n"
148 "movi v28.4s, #0x0\n"
149 "movi v27.4s, #0x0\n"
150 "ld1r { v26.2d }, [x21], #0x8\n"
151 "ldr q25, [x24, #0x20]\n"
152 "sub x20, x20, #0x1\n"
153 "ldr q24, [x24, #0x30]\n"
154 "fcvtl v31.4s, v31.4h\n"
155 "fcvtl v23.4s, v16.4h\n"
156 "add x24, x24, #0x40\n"
157 "ld1r { v22.2d }, [x21], #0x8\n"
158 "shl v21.16b, v30.16b, #0x4\n"
159 "shl v20.16b, v29.16b, #0x4\n"
160 "ld1r { v19.2d }, [x21], #0x8\n"
161 "ld1r { v18.2d }, [x21], #0x8\n"
162 "shl v17.16b, v25.16b, #0x4\n"
163 "and v30.16b, v30.16b, v1.16b\n"
164 "shl v16.16b, v24.16b, #0x4\n"
165 "and v29.16b, v29.16b, v1.16b\n"
166 ".inst 0x4e9a96bc // sdot v28.4s, v21.16b, v26.16b\n"
167 ".inst 0x4e9a969b // sdot v27.4s, v20.16b, v26.16b\n"
168 "and v25.16b, v25.16b, v1.16b\n"
169 "and v24.16b, v24.16b, v1.16b\n"
170 "fmul v23.4s, v23.4s, v31.4s\n"
171 ".inst 0x4e96963c // sdot v28.4s, v17.16b, v22.16b\n"
172 ".inst 0x4e96961b // sdot v27.4s, v16.16b, v22.16b\n"
173 ".inst 0x4e9397dc // sdot v28.4s, v30.16b, v19.16b\n"
174 ".inst 0x4e9397bb // sdot v27.4s, v29.16b, v19.16b\n"
175 ".inst 0x4e92973c // sdot v28.4s, v25.16b, v18.16b\n"
176 ".inst 0x4e92971b // sdot v27.4s, v24.16b, v18.16b\n"
177 "addp v28.4s, v28.4s, v27.4s\n"
178 "scvtf v28.4s, v28.4s, #0x4\n"
179 "fmla v0.4s, v28.4s, v23.4s\n"
180 "cbnz x20, 3b\n"
181 "ld1r { v17.4s }, [%x[clamp_vals]]\n"
182 "add x20, %x[clamp_vals], #0x4\n"
183 "cmp x23, #0x4\n"
184 "ld1r { v16.4s }, [x20]\n"
185 "fmax v0.4s, v0.4s, v17.4s\n"
186 "fmin v0.4s, v0.4s, v16.4s\n"
187 "blt 4f\n"
188 "str q0, [%x[dst], #0x0]\n"
189 "b 7f\n"
190 "4:" // Partial output
191 "mov x20, %x[dst]\n"
192 "tbz x23, #1, 5f\n"
193 "st1 { v0.d }[0], [x20], #0x8\n"
194 "tbz x23, #0, 6f\n"
195 "st1 { v0.s }[2], [x20]\n"
196 "b 6f\n"
197 "5:" // Output block 0: partial_1_0
198 "st1 { v0.s }[0], [x20]\n"
199 "6:" // Output block 0: Done
200 "7:" // Stores done
201 "subs x23, x23, #0x4\n"
202 "add %x[dst], %x[dst], #0x10\n"
203 "bgt 2b\n"
204 "subs x25, x25, #0x1\n"
205 "add %x[lhs_packed], %x[lhs_packed], x26\n"
206 "mov %x[dst], x22\n"
207 "bgt 1b\n"
208 : [dst] "+&r"(dst), [lhs_packed] "+&r"(lhs_packed)
209 78 : [clamp_vals] "r"(clamp_vals), [dst_stride_row] "r"(dst_stride_row), [m] "r"(m), [n] "r"(n),
210 78 [num_blocks] "r"(num_blocks), [rhs_packed] "r"(rhs_packed)
211 : "cc", "memory", "v0", "v1", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26",
212 "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23", "x24", "x25", "x26");
213 78 }
214
215 #endif // Architectural feature check
216