KleidiAI Coverage Report


Directory: ./
File: kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi4cxp/kai_matmul_clamp_f32_qai8dxp4x8_qsi4cxp8x8_8x8x32_neon_i8mm.c
Date: 2025-10-20 13:18:31
Coverage Exec Excl Total
Lines: 97.6% 41 7 49
Functions: 100.0% 14 0 14
Branches: 50.0% 1 14 16

Line Branch Exec Source
1 //
2 // SPDX-FileCopyrightText: Copyright 2024-2025 Arm Limited and/or its affiliates <open-source-office@arm.com>
3 //
4 // SPDX-License-Identifier: Apache-2.0
5 //
6
7 // Do not flag up inline assembly blocks
8 #pragma GCC diagnostic ignored "-Woverlength-strings"
9
10 #if !defined(__ARM_FEATURE_MATMUL_INT8)
11 #error "I8mm extension required to compile this micro-kernel"
12 #else
13 #include "kai_matmul_clamp_f32_qai8dxp4x8_qsi4cxp8x8_8x8x32_neon_i8mm.h"
14
15 #include <arm_neon.h>
16 #include <stddef.h>
17 #include <stdint.h>
18
19 #include "kai/kai_common.h"
20
21 static const size_t kai_m_step = 8;
22 static const size_t kai_n_step = 8;
23 static const size_t kai_mr = 4;
24 static const size_t kai_nr = 8;
25 static const size_t kai_kr = 16;
26 static const size_t kai_sr = 2;
27 static const size_t kai_num_bytes_multiplier_lhs = sizeof(float);
28 static const size_t kai_num_bytes_multiplier_rhs = sizeof(float);
29 static const size_t kai_num_bytes_offset_lhs = sizeof(int32_t);
30 static const size_t kai_num_bytes_sum_rhs = sizeof(int32_t);
31 static const size_t kai_num_bytes_bias = sizeof(float);
32
33 641 inline static size_t kai_k_roundedup(size_t k) {
34 // Since we pack a float and int32 value at the end of the row,
35 // we must make sure that k is a multiple of 4 for alignment
36 641 size_t kr_sr_roundedup4 = kai_roundup(kai_kr * kai_sr, 4);
37 1282 return kai_roundup(k, kr_sr_roundedup4);
38 641 }
39
40 240 inline static size_t kai_lhs_packed_stride(size_t k) {
41 240 const size_t k_internal = kai_k_roundedup(k);
42
43 KAI_ASSERT((k_internal % 2) == 0);
44
45 480 return kai_mr * (k_internal * sizeof(int8_t) + kai_num_bytes_multiplier_lhs + kai_num_bytes_offset_lhs);
46 240 }
47
48 240 inline static size_t kai_rhs_packed_stride(size_t k) {
49 240 const size_t k_internal = kai_k_roundedup(k);
50
51 KAI_ASSERT((k_internal % 2) == 0);
52
53 480 return kai_nr * ((k_internal / 2) + kai_num_bytes_multiplier_rhs + kai_num_bytes_sum_rhs + kai_num_bytes_bias);
54 240 }
55
56 320 size_t kai_get_m_step_matmul_clamp_f32_qai8dxp4x8_qsi4cxp8x8_8x8x32_neon_i8mm(void) {
57 320 return kai_m_step;
58 }
59
60 320 size_t kai_get_n_step_matmul_clamp_f32_qai8dxp4x8_qsi4cxp8x8_8x8x32_neon_i8mm(void) {
61 320 return kai_n_step;
62 }
63
64 240 size_t kai_get_mr_matmul_clamp_f32_qai8dxp4x8_qsi4cxp8x8_8x8x32_neon_i8mm(void) {
65 240 return kai_mr;
66 }
67
68 240 size_t kai_get_nr_matmul_clamp_f32_qai8dxp4x8_qsi4cxp8x8_8x8x32_neon_i8mm(void) {
69 240 return kai_nr;
70 }
71
72 320 size_t kai_get_kr_matmul_clamp_f32_qai8dxp4x8_qsi4cxp8x8_8x8x32_neon_i8mm(void) {
73 320 return kai_kr;
74 }
75
76 320 size_t kai_get_sr_matmul_clamp_f32_qai8dxp4x8_qsi4cxp8x8_8x8x32_neon_i8mm(void) {
77 320 return kai_sr;
78 }
79
80 240 size_t kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp4x8_qsi4cxp8x8_8x8x32_neon_i8mm(size_t m_idx, size_t k) {
81 KAI_ASSERT((m_idx % kai_m_step) == 0);
82
83 240 return (m_idx / kai_mr) * kai_lhs_packed_stride(k);
84 }
85
86 240 size_t kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp4x8_qsi4cxp8x8_8x8x32_neon_i8mm(size_t n_idx, size_t k) {
87 KAI_ASSERT((n_idx % kai_n_step) == 0);
88
89 240 return (n_idx / kai_nr) * kai_rhs_packed_stride(k);
90 }
91
92 160 size_t kai_get_dst_offset_matmul_clamp_f32_qai8dxp4x8_qsi4cxp8x8_8x8x32_neon_i8mm(
93 size_t m_idx, size_t n_idx, size_t dst_stride) {
94 KAI_ASSERT((m_idx % kai_m_step) == 0);
95 KAI_ASSERT((n_idx % kai_n_step) == 0);
96
97 160 return (n_idx * sizeof(float)) + m_idx * dst_stride;
98 }
99
100 160 size_t kai_get_dst_size_matmul_clamp_f32_qai8dxp4x8_qsi4cxp8x8_8x8x32_neon_i8mm(size_t m, size_t n) {
101 160 return m * n * sizeof(float);
102 }
103
104 161 void kai_run_matmul_clamp_f32_qai8dxp4x8_qsi4cxp8x8_8x8x32_neon_i8mm(
105 size_t m, size_t n, size_t k, const void* lhs_packed, const void* rhs_packed,
106 float* dst, // NOLINT(readability-non-const-parameter)
107 size_t dst_stride_row, size_t dst_stride_col, float scalar_min, float scalar_max) {
108 KAI_ASSERT(dst_stride_col == sizeof(float));
109
110
1/2
✓ Branch 0 taken 161 times.
✗ Branch 1 not taken.
161 if (m == 0) {
111 return;
112 }
113
114 161 const size_t k_internal = kai_k_roundedup(k);
115
116 161 size_t num_blocks = k_internal / 32;
117
118 161 float clamp_vals[2] = {scalar_min, scalar_max};
119
120 322 __asm__ __volatile__(
121 "mov x12, %x[m]\n"
122 "mov x11, #0x80\n"
123 "movi v24.16b, #0xf0\n"
124 "mov x20, #0x20\n"
125 "cmp x12, #0x8\n"
126 "madd x11, %x[num_blocks], x11, x20\n"
127 "blt 12f\n"
128 "1:" // Row loop
129 "mov x10, %x[rhs_packed]\n"
130 "mov x9, %x[n]\n"
131 "add x28, %x[dst], %x[dst_stride_row], LSL #3\n"
132 "2:" // Column loop
133 "mov x22, %x[lhs_packed]\n"
134 "movi v13.4s, #0x0\n"
135 "movi v22.4s, #0x0\n"
136 "mov x21, %x[num_blocks]\n"
137 "movi v11.4s, #0x0\n"
138 "movi v15.4s, #0x0\n"
139 "movi v14.4s, #0x0\n"
140 "movi v17.4s, #0x0\n"
141 "add x20, x22, x11\n"
142 "movi v1.4s, #0x0\n"
143 "movi v8.4s, #0x0\n"
144 "movi v30.4s, #0x0\n"
145 "movi v28.4s, #0x0\n"
146 "movi v10.4s, #0x0\n"
147 "movi v27.4s, #0x0\n"
148 "movi v18.4s, #0x0\n"
149 "movi v12.4s, #0x0\n"
150 "movi v4.4s, #0x0\n"
151 "movi v26.4s, #0x0\n"
152 "3:" // Sub block loop
153 "ldr q31, [x10, #0x0]\n"
154 "ldr q3, [x10, #0x10]\n"
155 "subs x21, x21, #0x1\n"
156 "ldr q6, [x10, #0x20]\n"
157 "ldr q25, [x10, #0x30]\n"
158 "ldr q19, [x22, #0x0]\n"
159 "ldr q7, [x22, #0x10]\n"
160 "ldr q20, [x20, #0x0]\n"
161 "ldr q5, [x20, #0x10]\n"
162 "shl v21.16b, v31.16b, #0x4\n"
163 "shl v2.16b, v3.16b, #0x4\n"
164 "ldr q16, [x10, #0x40]\n"
165 "ldr q23, [x10, #0x50]\n"
166 "shl v9.16b, v6.16b, #0x4\n"
167 "shl v29.16b, v25.16b, #0x4\n"
168 "ldr q0, [x10, #0x60]\n"
169 "and v31.16b, v31.16b, v24.16b\n"
170 "and v3.16b, v3.16b, v24.16b\n"
171 ".inst 0x4e95a66d // smmla v13.4s, v19.16b, v21.16b\n"
172 ".inst 0x4e82a66b // smmla v11.4s, v19.16b, v2.16b\n"
173 ".inst 0x4e95a4ee // smmla v14.4s, v7.16b, v21.16b\n"
174 "and v6.16b, v6.16b, v24.16b\n"
175 ".inst 0x4e89a676 // smmla v22.4s, v19.16b, v9.16b\n"
176 ".inst 0x4e9da66f // smmla v15.4s, v19.16b, v29.16b\n"
177 "ldr q19, [x10, #0x70]\n"
178 "and v25.16b, v25.16b, v24.16b\n"
179 ".inst 0x4e82a4e1 // smmla v1.4s, v7.16b, v2.16b\n"
180 ".inst 0x4e89a4f1 // smmla v17.4s, v7.16b, v9.16b\n"
181 "add x10, x10, #0x80\n"
182 ".inst 0x4e9da4e8 // smmla v8.4s, v7.16b, v29.16b\n"
183 "ldr q7, [x22, #0x20]\n"
184 ".inst 0x4e95a69e // smmla v30.4s, v20.16b, v21.16b\n"
185 ".inst 0x4e82a68a // smmla v10.4s, v20.16b, v2.16b\n"
186 ".inst 0x4e89a69c // smmla v28.4s, v20.16b, v9.16b\n"
187 ".inst 0x4e9da69b // smmla v27.4s, v20.16b, v29.16b\n"
188 "ldr q20, [x22, #0x30]\n"
189 ".inst 0x4e95a4b2 // smmla v18.4s, v5.16b, v21.16b\n"
190 "ldr q21, [x20, #0x20]\n"
191 ".inst 0x4e82a4a4 // smmla v4.4s, v5.16b, v2.16b\n"
192 "ldr q2, [x20, #0x30]\n"
193 ".inst 0x4e89a4ac // smmla v12.4s, v5.16b, v9.16b\n"
194 "ldr q9, [x22, #0x40]\n"
195 ".inst 0x4e9da4ba // smmla v26.4s, v5.16b, v29.16b\n"
196 "ldr q29, [x22, #0x50]\n"
197 "shl v5.16b, v16.16b, #0x4\n"
198 "and v16.16b, v16.16b, v24.16b\n"
199 ".inst 0x4e85a4ed // smmla v13.4s, v7.16b, v5.16b\n"
200 ".inst 0x4e85a68e // smmla v14.4s, v20.16b, v5.16b\n"
201 ".inst 0x4e85a6be // smmla v30.4s, v21.16b, v5.16b\n"
202 ".inst 0x4e85a452 // smmla v18.4s, v2.16b, v5.16b\n"
203 "shl v5.16b, v23.16b, #0x4\n"
204 "and v23.16b, v23.16b, v24.16b\n"
205 ".inst 0x4e85a4eb // smmla v11.4s, v7.16b, v5.16b\n"
206 ".inst 0x4e85a681 // smmla v1.4s, v20.16b, v5.16b\n"
207 ".inst 0x4e85a6aa // smmla v10.4s, v21.16b, v5.16b\n"
208 ".inst 0x4e85a444 // smmla v4.4s, v2.16b, v5.16b\n"
209 "shl v5.16b, v0.16b, #0x4\n"
210 ".inst 0x4e9fa52d // smmla v13.4s, v9.16b, v31.16b\n"
211 ".inst 0x4e9fa7ae // smmla v14.4s, v29.16b, v31.16b\n"
212 "and v0.16b, v0.16b, v24.16b\n"
213 ".inst 0x4e85a4f6 // smmla v22.4s, v7.16b, v5.16b\n"
214 ".inst 0x4e85a691 // smmla v17.4s, v20.16b, v5.16b\n"
215 ".inst 0x4e85a6bc // smmla v28.4s, v21.16b, v5.16b\n"
216 ".inst 0x4e85a44c // smmla v12.4s, v2.16b, v5.16b\n"
217 "shl v5.16b, v19.16b, #0x4\n"
218 ".inst 0x4e83a52b // smmla v11.4s, v9.16b, v3.16b\n"
219 ".inst 0x4e83a7a1 // smmla v1.4s, v29.16b, v3.16b\n"
220 "and v19.16b, v19.16b, v24.16b\n"
221 ".inst 0x4e85a4ef // smmla v15.4s, v7.16b, v5.16b\n"
222 "ldr q7, [x20, #0x40]\n"
223 ".inst 0x4e85a688 // smmla v8.4s, v20.16b, v5.16b\n"
224 "ldr q20, [x20, #0x50]\n"
225 ".inst 0x4e85a6bb // smmla v27.4s, v21.16b, v5.16b\n"
226 "ldr q21, [x22, #0x60]\n"
227 ".inst 0x4e85a45a // smmla v26.4s, v2.16b, v5.16b\n"
228 "ldr q5, [x22, #0x70]\n"
229 "ldr q2, [x20, #0x60]\n"
230 ".inst 0x4e86a536 // smmla v22.4s, v9.16b, v6.16b\n"
231 ".inst 0x4e86a7b1 // smmla v17.4s, v29.16b, v6.16b\n"
232 "add x22, x22, #0x80\n"
233 ".inst 0x4e9fa4fe // smmla v30.4s, v7.16b, v31.16b\n"
234 ".inst 0x4e83a4ea // smmla v10.4s, v7.16b, v3.16b\n"
235 ".inst 0x4e99a52f // smmla v15.4s, v9.16b, v25.16b\n"
236 "ldr q9, [x20, #0x70]\n"
237 ".inst 0x4e99a7a8 // smmla v8.4s, v29.16b, v25.16b\n"
238 "add x20, x20, #0x80\n"
239 ".inst 0x4e86a4fc // smmla v28.4s, v7.16b, v6.16b\n"
240 ".inst 0x4e99a4fb // smmla v27.4s, v7.16b, v25.16b\n"
241 ".inst 0x4e9fa692 // smmla v18.4s, v20.16b, v31.16b\n"
242 ".inst 0x4e83a684 // smmla v4.4s, v20.16b, v3.16b\n"
243 ".inst 0x4e86a68c // smmla v12.4s, v20.16b, v6.16b\n"
244 ".inst 0x4e99a69a // smmla v26.4s, v20.16b, v25.16b\n"
245 ".inst 0x4e90a6ad // smmla v13.4s, v21.16b, v16.16b\n"
246 ".inst 0x4e97a6ab // smmla v11.4s, v21.16b, v23.16b\n"
247 ".inst 0x4e80a6b6 // smmla v22.4s, v21.16b, v0.16b\n"
248 ".inst 0x4e93a6af // smmla v15.4s, v21.16b, v19.16b\n"
249 ".inst 0x4e90a4ae // smmla v14.4s, v5.16b, v16.16b\n"
250 ".inst 0x4e97a4a1 // smmla v1.4s, v5.16b, v23.16b\n"
251 ".inst 0x4e80a4b1 // smmla v17.4s, v5.16b, v0.16b\n"
252 ".inst 0x4e93a4a8 // smmla v8.4s, v5.16b, v19.16b\n"
253 ".inst 0x4e90a45e // smmla v30.4s, v2.16b, v16.16b\n"
254 ".inst 0x4e97a44a // smmla v10.4s, v2.16b, v23.16b\n"
255 ".inst 0x4e80a45c // smmla v28.4s, v2.16b, v0.16b\n"
256 ".inst 0x4e93a45b // smmla v27.4s, v2.16b, v19.16b\n"
257 ".inst 0x4e90a532 // smmla v18.4s, v9.16b, v16.16b\n"
258 ".inst 0x4e97a524 // smmla v4.4s, v9.16b, v23.16b\n"
259 ".inst 0x4e80a52c // smmla v12.4s, v9.16b, v0.16b\n"
260 ".inst 0x4e93a53a // smmla v26.4s, v9.16b, v19.16b\n"
261 "bgt 3b\n"
262 "ldr q5, [x10, #0x0]\n"
263 "ldr q19, [x10, #0x10]\n"
264 "uzp1 v2.2d, v13.2d, v11.2d\n"
265 "uzp2 v20.2d, v13.2d, v11.2d\n"
266 "ld1 { v11.4s }, [x22]\n"
267 "ldr q23, [x10, #0x20]\n"
268 "uzp1 v9.2d, v22.2d, v15.2d\n"
269 "uzp2 v29.2d, v22.2d, v15.2d\n"
270 "ldr q6, [x10, #0x30]\n"
271 "uzp1 v31.2d, v14.2d, v1.2d\n"
272 "uzp2 v7.2d, v14.2d, v1.2d\n"
273 "add x22, x22, #0x10\n"
274 "ldr q22, [x22, #0x0]\n"
275 "uzp1 v0.2d, v17.2d, v8.2d\n"
276 "uzp2 v16.2d, v17.2d, v8.2d\n"
277 "add x10, x10, #0x40\n"
278 "mla v2.4s, v5.4s, v11.s[0]\n"
279 "mla v9.4s, v19.4s, v11.s[0]\n"
280 "mla v20.4s, v5.4s, v11.s[1]\n"
281 "mla v29.4s, v19.4s, v11.s[1]\n"
282 "mla v31.4s, v5.4s, v11.s[2]\n"
283 "mla v0.4s, v19.4s, v11.s[2]\n"
284 "fmul v15.4s, v23.4s, v22.s[0]\n"
285 "mla v7.4s, v5.4s, v11.s[3]\n"
286 "mla v16.4s, v19.4s, v11.s[3]\n"
287 "fmul v11.4s, v6.4s, v22.s[0]\n"
288 "scvtf v2.4s, v2.4s\n"
289 "scvtf v9.4s, v9.4s\n"
290 "fmul v25.4s, v23.4s, v22.s[1]\n"
291 "scvtf v20.4s, v20.4s\n"
292 "fmul v14.4s, v6.4s, v22.s[1]\n"
293 "scvtf v29.4s, v29.4s\n"
294 "fmul v1.4s, v23.4s, v22.s[2]\n"
295 "scvtf v31.4s, v31.4s\n"
296 "fmul v17.4s, v6.4s, v22.s[2]\n"
297 "scvtf v0.4s, v0.4s\n"
298 "fmul v21.4s, v23.4s, v22.s[3]\n"
299 "scvtf v7.4s, v7.4s\n"
300 "fmul v3.4s, v6.4s, v22.s[3]\n"
301 "scvtf v16.4s, v16.4s\n"
302 "fmul v13.4s, v2.4s, v15.4s\n"
303 "fmul v22.4s, v9.4s, v11.4s\n"
304 "fmul v11.4s, v20.4s, v25.4s\n"
305 "fmul v15.4s, v29.4s, v14.4s\n"
306 "fmul v14.4s, v31.4s, v1.4s\n"
307 "fmul v17.4s, v0.4s, v17.4s\n"
308 "fmul v1.4s, v7.4s, v21.4s\n"
309 "fmul v8.4s, v16.4s, v3.4s\n"
310 "ld1 { v20.4s }, [x20]\n"
311 "uzp1 v2.2d, v30.2d, v10.2d\n"
312 "uzp2 v10.2d, v30.2d, v10.2d\n"
313 "add x20, x20, #0x10\n"
314 "ldr q3, [x20, #0x0]\n"
315 "uzp1 v0.2d, v28.2d, v27.2d\n"
316 "uzp2 v31.2d, v28.2d, v27.2d\n"
317 "uzp1 v29.2d, v18.2d, v4.2d\n"
318 "uzp2 v9.2d, v18.2d, v4.2d\n"
319 "uzp1 v4.2d, v12.2d, v26.2d\n"
320 "uzp2 v21.2d, v12.2d, v26.2d\n"
321 "mla v2.4s, v5.4s, v20.s[0]\n"
322 "mla v0.4s, v19.4s, v20.s[0]\n"
323 "mla v10.4s, v5.4s, v20.s[1]\n"
324 "fmul v30.4s, v23.4s, v3.s[0]\n"
325 "mla v31.4s, v19.4s, v20.s[1]\n"
326 "mla v29.4s, v5.4s, v20.s[2]\n"
327 "fmul v7.4s, v6.4s, v3.s[0]\n"
328 "mla v4.4s, v19.4s, v20.s[2]\n"
329 "mla v9.4s, v5.4s, v20.s[3]\n"
330 "fmul v18.4s, v23.4s, v3.s[1]\n"
331 "mla v21.4s, v19.4s, v20.s[3]\n"
332 "scvtf v2.4s, v2.4s\n"
333 "scvtf v0.4s, v0.4s\n"
334 "scvtf v10.4s, v10.4s\n"
335 "fmul v27.4s, v6.4s, v3.s[1]\n"
336 "scvtf v31.4s, v31.4s\n"
337 "fmul v20.4s, v23.4s, v3.s[2]\n"
338 "scvtf v29.4s, v29.4s\n"
339 "fmul v19.4s, v6.4s, v3.s[2]\n"
340 "scvtf v4.4s, v4.4s\n"
341 "fmul v23.4s, v23.4s, v3.s[3]\n"
342 "scvtf v9.4s, v9.4s\n"
343 "fmul v6.4s, v6.4s, v3.s[3]\n"
344 "scvtf v21.4s, v21.4s\n"
345 "fmul v30.4s, v2.4s, v30.4s\n"
346 "fmul v28.4s, v0.4s, v7.4s\n"
347 "fmul v10.4s, v10.4s, v18.4s\n"
348 "fmul v27.4s, v31.4s, v27.4s\n"
349 "fmul v18.4s, v29.4s, v20.4s\n"
350 "fmul v12.4s, v4.4s, v19.4s\n"
351 "fmul v4.4s, v9.4s, v23.4s\n"
352 "fmul v26.4s, v21.4s, v6.4s\n"
353 "ldr q20, [x10, #0x0]\n"
354 "ldr q19, [x10, #0x10]\n"
355 "add x20, %x[clamp_vals], #0x4\n"
356 "cmp x9, #0x8\n"
357 "ld1r { v9.4s }, [%x[clamp_vals]]\n"
358 "ld1r { v6.4s }, [x20]\n"
359 "add x10, x10, #0x20\n"
360 "fadd v13.4s, v13.4s, v20.4s\n"
361 "fadd v22.4s, v22.4s, v19.4s\n"
362 "fadd v11.4s, v11.4s, v20.4s\n"
363 "fadd v15.4s, v15.4s, v19.4s\n"
364 "fadd v14.4s, v14.4s, v20.4s\n"
365 "fadd v17.4s, v17.4s, v19.4s\n"
366 "fadd v1.4s, v1.4s, v20.4s\n"
367 "fadd v8.4s, v8.4s, v19.4s\n"
368 "fadd v30.4s, v30.4s, v20.4s\n"
369 "fadd v28.4s, v28.4s, v19.4s\n"
370 "fadd v10.4s, v10.4s, v20.4s\n"
371 "fadd v27.4s, v27.4s, v19.4s\n"
372 "fadd v18.4s, v18.4s, v20.4s\n"
373 "fadd v12.4s, v12.4s, v19.4s\n"
374 "fadd v4.4s, v4.4s, v20.4s\n"
375 "fadd v26.4s, v26.4s, v19.4s\n"
376 "fmax v13.4s, v13.4s, v9.4s\n"
377 "fmax v22.4s, v22.4s, v9.4s\n"
378 "fmax v11.4s, v11.4s, v9.4s\n"
379 "fmax v15.4s, v15.4s, v9.4s\n"
380 "fmax v14.4s, v14.4s, v9.4s\n"
381 "fmax v17.4s, v17.4s, v9.4s\n"
382 "fmax v1.4s, v1.4s, v9.4s\n"
383 "fmax v8.4s, v8.4s, v9.4s\n"
384 "fmax v30.4s, v30.4s, v9.4s\n"
385 "fmax v28.4s, v28.4s, v9.4s\n"
386 "fmax v10.4s, v10.4s, v9.4s\n"
387 "fmax v27.4s, v27.4s, v9.4s\n"
388 "fmax v18.4s, v18.4s, v9.4s\n"
389 "fmax v12.4s, v12.4s, v9.4s\n"
390 "fmax v4.4s, v4.4s, v9.4s\n"
391 "fmax v26.4s, v26.4s, v9.4s\n"
392 "fmin v13.4s, v13.4s, v6.4s\n"
393 "fmin v22.4s, v22.4s, v6.4s\n"
394 "fmin v11.4s, v11.4s, v6.4s\n"
395 "fmin v15.4s, v15.4s, v6.4s\n"
396 "fmin v14.4s, v14.4s, v6.4s\n"
397 "fmin v17.4s, v17.4s, v6.4s\n"
398 "fmin v1.4s, v1.4s, v6.4s\n"
399 "fmin v8.4s, v8.4s, v6.4s\n"
400 "fmin v30.4s, v30.4s, v6.4s\n"
401 "fmin v28.4s, v28.4s, v6.4s\n"
402 "fmin v10.4s, v10.4s, v6.4s\n"
403 "fmin v27.4s, v27.4s, v6.4s\n"
404 "fmin v18.4s, v18.4s, v6.4s\n"
405 "fmin v12.4s, v12.4s, v6.4s\n"
406 "fmin v4.4s, v4.4s, v6.4s\n"
407 "fmin v26.4s, v26.4s, v6.4s\n"
408 "blt 6f\n"
409 "mov x20, %x[dst]\n"
410 "str q13, [x20, #0x0]\n"
411 "str q22, [x20, #0x10]\n"
412 "add x20, x20, %x[dst_stride_row]\n"
413 "str q11, [x20, #0x0]\n"
414 "str q15, [x20, #0x10]\n"
415 "add x20, x20, %x[dst_stride_row]\n"
416 "str q14, [x20, #0x0]\n"
417 "str q17, [x20, #0x10]\n"
418 "add x20, x20, %x[dst_stride_row]\n"
419 "str q1, [x20, #0x0]\n"
420 "str q8, [x20, #0x10]\n"
421 "add x20, x20, %x[dst_stride_row]\n"
422 "str q30, [x20, #0x0]\n"
423 "str q28, [x20, #0x10]\n"
424 "add x20, x20, %x[dst_stride_row]\n"
425 "str q10, [x20, #0x0]\n"
426 "str q27, [x20, #0x10]\n"
427 "add x20, x20, %x[dst_stride_row]\n"
428 "str q18, [x20, #0x0]\n"
429 "str q12, [x20, #0x10]\n"
430 "add x20, x20, %x[dst_stride_row]\n"
431 "str q4, [x20, #0x0]\n"
432 "str q26, [x20, #0x10]\n"
433 "b 11f\n"
434 "6:" // Partial output
435 "mov x27, %x[dst]\n"
436 "add x26, x27, %x[dst_stride_row], LSL #2\n"
437 "add x25, x26, %x[dst_stride_row], LSL #1\n"
438 "add x24, x26, %x[dst_stride_row]\n"
439 "add x23, x25, %x[dst_stride_row]\n"
440 "add x22, x27, %x[dst_stride_row], LSL #1\n"
441 "add x21, x27, %x[dst_stride_row]\n"
442 "add x20, x22, %x[dst_stride_row]\n"
443 "tbz x9, #2, 8f\n"
444 "st1 { v4.4s }, [x23], #0x10\n"
445 "st1 { v18.4s }, [x25], #0x10\n"
446 "st1 { v10.4s }, [x24], #0x10\n"
447 "st1 { v30.4s }, [x26], #0x10\n"
448 "st1 { v1.4s }, [x20], #0x10\n"
449 "st1 { v14.4s }, [x22], #0x10\n"
450 "st1 { v11.4s }, [x21], #0x10\n"
451 "st1 { v13.4s }, [x27], #0x10\n"
452 "tbz x9, #1, 7f\n"
453 "st1 { v26.d }[0], [x23], #0x8\n"
454 "st1 { v12.d }[0], [x25], #0x8\n"
455 "st1 { v27.d }[0], [x24], #0x8\n"
456 "st1 { v28.d }[0], [x26], #0x8\n"
457 "st1 { v8.d }[0], [x20], #0x8\n"
458 "st1 { v17.d }[0], [x22], #0x8\n"
459 "st1 { v15.d }[0], [x21], #0x8\n"
460 "st1 { v22.d }[0], [x27], #0x8\n"
461 "tbz x9, #0, 10f\n"
462 "st1 { v26.s }[2], [x23]\n"
463 "st1 { v12.s }[2], [x25]\n"
464 "st1 { v27.s }[2], [x24]\n"
465 "st1 { v28.s }[2], [x26]\n"
466 "st1 { v8.s }[2], [x20]\n"
467 "st1 { v17.s }[2], [x22]\n"
468 "st1 { v15.s }[2], [x21]\n"
469 "st1 { v22.s }[2], [x27]\n"
470 "b 10f\n"
471 "7:" // Output block 0: partial_1_4
472 "tbz x9, #0, 10f\n"
473 "st1 { v26.s }[0], [x23]\n"
474 "st1 { v12.s }[0], [x25]\n"
475 "st1 { v27.s }[0], [x24]\n"
476 "st1 { v28.s }[0], [x26]\n"
477 "st1 { v8.s }[0], [x20]\n"
478 "st1 { v17.s }[0], [x22]\n"
479 "st1 { v15.s }[0], [x21]\n"
480 "st1 { v22.s }[0], [x27]\n"
481 "b 10f\n"
482 "8:" // Output block 0: partial_2_0
483 "tbz x9, #1, 9f\n"
484 "st1 { v4.d }[0], [x23], #0x8\n"
485 "st1 { v18.d }[0], [x25], #0x8\n"
486 "st1 { v10.d }[0], [x24], #0x8\n"
487 "st1 { v30.d }[0], [x26], #0x8\n"
488 "st1 { v1.d }[0], [x20], #0x8\n"
489 "st1 { v14.d }[0], [x22], #0x8\n"
490 "st1 { v11.d }[0], [x21], #0x8\n"
491 "st1 { v13.d }[0], [x27], #0x8\n"
492 "tbz x9, #0, 10f\n"
493 "st1 { v4.s }[2], [x23]\n"
494 "st1 { v18.s }[2], [x25]\n"
495 "st1 { v10.s }[2], [x24]\n"
496 "st1 { v30.s }[2], [x26]\n"
497 "st1 { v1.s }[2], [x20]\n"
498 "st1 { v14.s }[2], [x22]\n"
499 "st1 { v11.s }[2], [x21]\n"
500 "st1 { v13.s }[2], [x27]\n"
501 "b 10f\n"
502 "9:" // Output block 0: partial_1_0
503 "st1 { v4.s }[0], [x23]\n"
504 "st1 { v18.s }[0], [x25]\n"
505 "st1 { v10.s }[0], [x24]\n"
506 "st1 { v30.s }[0], [x26]\n"
507 "st1 { v1.s }[0], [x20]\n"
508 "st1 { v14.s }[0], [x22]\n"
509 "st1 { v11.s }[0], [x21]\n"
510 "st1 { v13.s }[0], [x27]\n"
511 "10:" // Output block 0: Done
512 "11:" // Output stage exit
513 "subs x9, x9, #0x8\n"
514 "add %x[dst], %x[dst], #0x20\n"
515 "bgt 2b\n"
516 "mov x20, #0x2\n"
517 "sub x12, x12, #0x8\n"
518 "cmp x12, #0x8\n"
519 "mov %x[dst], x28\n"
520 "madd %x[lhs_packed], x20, x11, %x[lhs_packed]\n"
521 "bge 1b\n"
522 "12:" // Row loop skip
523 "cbz x12, 23f\n"
524 "13:" // Row tail: Row loop
525 "mov x26, %x[rhs_packed]\n"
526 "mov x25, %x[n]\n"
527 "add x24, %x[dst], %x[dst_stride_row], LSL #2\n"
528 "14:" // Row tail: Column loop
529 "mov x22, %x[lhs_packed]\n"
530 "movi v13.4s, #0x0\n"
531 "movi v22.4s, #0x0\n"
532 "mov x20, %x[num_blocks]\n"
533 "movi v11.4s, #0x0\n"
534 "movi v15.4s, #0x0\n"
535 "movi v14.4s, #0x0\n"
536 "movi v17.4s, #0x0\n"
537 "movi v1.4s, #0x0\n"
538 "movi v8.4s, #0x0\n"
539 "15:" // Row tail: Sub block loop
540 "ldr q16, [x26, #0x0]\n"
541 "ldr q7, [x26, #0x10]\n"
542 "subs x20, x20, #0x1\n"
543 "ldr q6, [x26, #0x20]\n"
544 "ldr q5, [x26, #0x30]\n"
545 "ldr q4, [x22, #0x0]\n"
546 "ldr q9, [x22, #0x10]\n"
547 "ldr q10, [x26, #0x40]\n"
548 "ldr q3, [x26, #0x50]\n"
549 "shl v0.16b, v16.16b, #0x4\n"
550 "shl v19.16b, v7.16b, #0x4\n"
551 "ldr q31, [x26, #0x60]\n"
552 "ldr q27, [x26, #0x70]\n"
553 "shl v18.16b, v6.16b, #0x4\n"
554 "shl v12.16b, v5.16b, #0x4\n"
555 "ldr q29, [x22, #0x20]\n"
556 "ldr q28, [x22, #0x30]\n"
557 "and v16.16b, v16.16b, v24.16b\n"
558 "and v7.16b, v7.16b, v24.16b\n"
559 "ldr q2, [x22, #0x40]\n"
560 "ldr q23, [x22, #0x50]\n"
561 ".inst 0x4e80a48d // smmla v13.4s, v4.16b, v0.16b\n"
562 ".inst 0x4e93a48b // smmla v11.4s, v4.16b, v19.16b\n"
563 "ldr q30, [x22, #0x60]\n"
564 "ldr q21, [x22, #0x70]\n"
565 ".inst 0x4e92a496 // smmla v22.4s, v4.16b, v18.16b\n"
566 ".inst 0x4e8ca48f // smmla v15.4s, v4.16b, v12.16b\n"
567 ".inst 0x4e80a52e // smmla v14.4s, v9.16b, v0.16b\n"
568 ".inst 0x4e93a521 // smmla v1.4s, v9.16b, v19.16b\n"
569 "shl v20.16b, v10.16b, #0x4\n"
570 "add x26, x26, #0x80\n"
571 ".inst 0x4e92a531 // smmla v17.4s, v9.16b, v18.16b\n"
572 ".inst 0x4e8ca528 // smmla v8.4s, v9.16b, v12.16b\n"
573 "shl v19.16b, v3.16b, #0x4\n"
574 "add x22, x22, #0x80\n"
575 "shl v18.16b, v31.16b, #0x4\n"
576 "shl v12.16b, v27.16b, #0x4\n"
577 ".inst 0x4e94a7ad // smmla v13.4s, v29.16b, v20.16b\n"
578 "and v6.16b, v6.16b, v24.16b\n"
579 "and v5.16b, v5.16b, v24.16b\n"
580 ".inst 0x4e93a7ab // smmla v11.4s, v29.16b, v19.16b\n"
581 ".inst 0x4e94a78e // smmla v14.4s, v28.16b, v20.16b\n"
582 ".inst 0x4e93a781 // smmla v1.4s, v28.16b, v19.16b\n"
583 "and v10.16b, v10.16b, v24.16b\n"
584 ".inst 0x4e92a7b6 // smmla v22.4s, v29.16b, v18.16b\n"
585 ".inst 0x4e8ca7af // smmla v15.4s, v29.16b, v12.16b\n"
586 "and v3.16b, v3.16b, v24.16b\n"
587 ".inst 0x4e92a791 // smmla v17.4s, v28.16b, v18.16b\n"
588 ".inst 0x4e8ca788 // smmla v8.4s, v28.16b, v12.16b\n"
589 "and v31.16b, v31.16b, v24.16b\n"
590 ".inst 0x4e90a44d // smmla v13.4s, v2.16b, v16.16b\n"
591 ".inst 0x4e87a44b // smmla v11.4s, v2.16b, v7.16b\n"
592 "and v27.16b, v27.16b, v24.16b\n"
593 ".inst 0x4e90a6ee // smmla v14.4s, v23.16b, v16.16b\n"
594 ".inst 0x4e87a6e1 // smmla v1.4s, v23.16b, v7.16b\n"
595 ".inst 0x4e86a456 // smmla v22.4s, v2.16b, v6.16b\n"
596 ".inst 0x4e85a44f // smmla v15.4s, v2.16b, v5.16b\n"
597 ".inst 0x4e86a6f1 // smmla v17.4s, v23.16b, v6.16b\n"
598 ".inst 0x4e85a6e8 // smmla v8.4s, v23.16b, v5.16b\n"
599 ".inst 0x4e8aa7cd // smmla v13.4s, v30.16b, v10.16b\n"
600 ".inst 0x4e83a7cb // smmla v11.4s, v30.16b, v3.16b\n"
601 ".inst 0x4e8aa6ae // smmla v14.4s, v21.16b, v10.16b\n"
602 ".inst 0x4e83a6a1 // smmla v1.4s, v21.16b, v3.16b\n"
603 ".inst 0x4e9fa7d6 // smmla v22.4s, v30.16b, v31.16b\n"
604 ".inst 0x4e9ba7cf // smmla v15.4s, v30.16b, v27.16b\n"
605 ".inst 0x4e9fa6b1 // smmla v17.4s, v21.16b, v31.16b\n"
606 ".inst 0x4e9ba6a8 // smmla v8.4s, v21.16b, v27.16b\n"
607 "bgt 15b\n"
608 "ldr q21, [x26, #0x0]\n"
609 "ldr q20, [x26, #0x10]\n"
610 "uzp1 v9.2d, v13.2d, v11.2d\n"
611 "uzp2 v2.2d, v13.2d, v11.2d\n"
612 "ld1 { v19.4s }, [x22]\n"
613 "ldr q27, [x26, #0x20]\n"
614 "uzp1 v0.2d, v22.2d, v15.2d\n"
615 "uzp2 v31.2d, v22.2d, v15.2d\n"
616 "ldr q13, [x26, #0x30]\n"
617 "uzp1 v29.2d, v14.2d, v1.2d\n"
618 "uzp2 v10.2d, v14.2d, v1.2d\n"
619 "add x22, x22, #0x10\n"
620 "ldr q23, [x22, #0x0]\n"
621 "uzp1 v5.2d, v17.2d, v8.2d\n"
622 "uzp2 v18.2d, v17.2d, v8.2d\n"
623 "add x26, x26, #0x40\n"
624 "mla v9.4s, v21.4s, v19.s[0]\n"
625 "mla v0.4s, v20.4s, v19.s[0]\n"
626 "mla v2.4s, v21.4s, v19.s[1]\n"
627 "mla v31.4s, v20.4s, v19.s[1]\n"
628 "mla v29.4s, v21.4s, v19.s[2]\n"
629 "mla v5.4s, v20.4s, v19.s[2]\n"
630 "fmul v30.4s, v27.4s, v23.s[0]\n"
631 "mla v10.4s, v21.4s, v19.s[3]\n"
632 "mla v18.4s, v20.4s, v19.s[3]\n"
633 "fmul v17.4s, v13.4s, v23.s[0]\n"
634 "scvtf v9.4s, v9.4s\n"
635 "scvtf v0.4s, v0.4s\n"
636 "fmul v21.4s, v27.4s, v23.s[1]\n"
637 "scvtf v2.4s, v2.4s\n"
638 "fmul v20.4s, v13.4s, v23.s[1]\n"
639 "scvtf v31.4s, v31.4s\n"
640 "fmul v19.4s, v27.4s, v23.s[2]\n"
641 "scvtf v29.4s, v29.4s\n"
642 "fmul v28.4s, v13.4s, v23.s[2]\n"
643 "scvtf v5.4s, v5.4s\n"
644 "fmul v26.4s, v27.4s, v23.s[3]\n"
645 "scvtf v10.4s, v10.4s\n"
646 "fmul v16.4s, v13.4s, v23.s[3]\n"
647 "scvtf v18.4s, v18.4s\n"
648 "fmul v13.4s, v9.4s, v30.4s\n"
649 "fmul v22.4s, v0.4s, v17.4s\n"
650 "fmul v11.4s, v2.4s, v21.4s\n"
651 "fmul v15.4s, v31.4s, v20.4s\n"
652 "fmul v14.4s, v29.4s, v19.4s\n"
653 "fmul v17.4s, v5.4s, v28.4s\n"
654 "fmul v1.4s, v10.4s, v26.4s\n"
655 "fmul v8.4s, v18.4s, v16.4s\n"
656 "ldr q19, [x26, #0x0]\n"
657 "ldr q18, [x26, #0x10]\n"
658 "add x20, %x[clamp_vals], #0x4\n"
659 "cmp x25, #0x8\n"
660 "ld1r { v20.4s }, [%x[clamp_vals]]\n"
661 "ld1r { v27.4s }, [x20]\n"
662 "add x26, x26, #0x20\n"
663 "fadd v13.4s, v13.4s, v19.4s\n"
664 "fadd v22.4s, v22.4s, v18.4s\n"
665 "fadd v11.4s, v11.4s, v19.4s\n"
666 "fadd v15.4s, v15.4s, v18.4s\n"
667 "fadd v14.4s, v14.4s, v19.4s\n"
668 "fadd v17.4s, v17.4s, v18.4s\n"
669 "fadd v1.4s, v1.4s, v19.4s\n"
670 "fadd v8.4s, v8.4s, v18.4s\n"
671 "fmax v13.4s, v13.4s, v20.4s\n"
672 "fmax v22.4s, v22.4s, v20.4s\n"
673 "fmax v11.4s, v11.4s, v20.4s\n"
674 "fmax v15.4s, v15.4s, v20.4s\n"
675 "fmax v14.4s, v14.4s, v20.4s\n"
676 "fmax v17.4s, v17.4s, v20.4s\n"
677 "fmax v1.4s, v1.4s, v20.4s\n"
678 "fmax v8.4s, v8.4s, v20.4s\n"
679 "fmin v13.4s, v13.4s, v27.4s\n"
680 "fmin v22.4s, v22.4s, v27.4s\n"
681 "fmin v11.4s, v11.4s, v27.4s\n"
682 "fmin v15.4s, v15.4s, v27.4s\n"
683 "fmin v14.4s, v14.4s, v27.4s\n"
684 "fmin v17.4s, v17.4s, v27.4s\n"
685 "fmin v1.4s, v1.4s, v27.4s\n"
686 "fmin v8.4s, v8.4s, v27.4s\n"
687 "blt 17f\n"
688 "mov x20, %x[dst]\n"
689 "cmp x12, #0x1\n"
690 "str q13, [x20, #0x0]\n"
691 "str q22, [x20, #0x10]\n"
692 "add x20, x20, %x[dst_stride_row]\n"
693 "ble 22f\n"
694 "cmp x12, #0x2\n"
695 "str q11, [x20, #0x0]\n"
696 "str q15, [x20, #0x10]\n"
697 "add x20, x20, %x[dst_stride_row]\n"
698 "ble 22f\n"
699 "cmp x12, #0x3\n"
700 "str q14, [x20, #0x0]\n"
701 "str q17, [x20, #0x10]\n"
702 "add x20, x20, %x[dst_stride_row]\n"
703 "ble 22f\n"
704 "str q1, [x20, #0x0]\n"
705 "str q8, [x20, #0x10]\n"
706 "b 22f\n"
707 "17:" // Row tail: Partial output
708 "mov x23, %x[dst]\n"
709 "cmp x12, #0x1\n"
710 "add x22, x23, %x[dst_stride_row]\n"
711 "csel x22, x22, x23, GT\n"
712 "cmp x12, #0x2\n"
713 "add x21, x23, %x[dst_stride_row], LSL #1\n"
714 "csel x21, x21, x22, GT\n"
715 "cmp x12, #0x3\n"
716 "add x20, x21, %x[dst_stride_row]\n"
717 "csel x20, x20, x21, GT\n"
718 "tbz x25, #2, 19f\n"
719 "st1 { v1.4s }, [x20], #0x10\n"
720 "st1 { v14.4s }, [x21], #0x10\n"
721 "st1 { v11.4s }, [x22], #0x10\n"
722 "st1 { v13.4s }, [x23], #0x10\n"
723 "tbz x25, #1, 18f\n"
724 "st1 { v8.d }[0], [x20], #0x8\n"
725 "st1 { v17.d }[0], [x21], #0x8\n"
726 "st1 { v15.d }[0], [x22], #0x8\n"
727 "st1 { v22.d }[0], [x23], #0x8\n"
728 "tbz x25, #0, 21f\n"
729 "st1 { v8.s }[2], [x20]\n"
730 "st1 { v17.s }[2], [x21]\n"
731 "st1 { v15.s }[2], [x22]\n"
732 "st1 { v22.s }[2], [x23]\n"
733 "b 21f\n"
734 "18:" // Row tail: Output block 0: partial_1_4
735 "tbz x25, #0, 21f\n"
736 "st1 { v8.s }[0], [x20]\n"
737 "st1 { v17.s }[0], [x21]\n"
738 "st1 { v15.s }[0], [x22]\n"
739 "st1 { v22.s }[0], [x23]\n"
740 "b 21f\n"
741 "19:" // Row tail: Output block 0: partial_2_0
742 "tbz x25, #1, 20f\n"
743 "st1 { v1.d }[0], [x20], #0x8\n"
744 "st1 { v14.d }[0], [x21], #0x8\n"
745 "st1 { v11.d }[0], [x22], #0x8\n"
746 "st1 { v13.d }[0], [x23], #0x8\n"
747 "tbz x25, #0, 21f\n"
748 "st1 { v1.s }[2], [x20]\n"
749 "st1 { v14.s }[2], [x21]\n"
750 "st1 { v11.s }[2], [x22]\n"
751 "st1 { v13.s }[2], [x23]\n"
752 "b 21f\n"
753 "20:" // Row tail: Output block 0: partial_1_0
754 "st1 { v1.s }[0], [x20]\n"
755 "st1 { v14.s }[0], [x21]\n"
756 "st1 { v11.s }[0], [x22]\n"
757 "st1 { v13.s }[0], [x23]\n"
758 "21:" // Row tail: Output block 0: Done
759 "22:" // Row tail: Output stage exit
760 "subs x25, x25, #0x8\n"
761 "add %x[dst], %x[dst], #0x20\n"
762 "bgt 14b\n"
763 "subs x12, x12, #0x4\n"
764 "add %x[lhs_packed], %x[lhs_packed], x11\n"
765 "mov %x[dst], x24\n"
766 "bgt 13b\n"
767 "23:" // Row tail: Row loop skip
768 : [dst] "+&r"(dst), [lhs_packed] "+&r"(lhs_packed)
769 161 : [clamp_vals] "r"(clamp_vals), [dst_stride_row] "r"(dst_stride_row), [m] "r"(m), [n] "r"(n),
770 161 [num_blocks] "r"(num_blocks), [rhs_packed] "r"(rhs_packed)
771 : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
772 "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29",
773 "v30", "v31", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28");
774 161 }
775 #endif // Architectural feature check
776