KleidiAI Coverage Report


Directory: ./
File: kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_8x4x32_neon_i8mm.c
Date: 2025-10-20 13:18:31
Coverage Exec Excl Total
Lines: 97.6% 41 19 61
Functions: 100.0% 16 0 16
Branches: 50.0% 1 38 40

Line Branch Exec Source
1 //
2 // SPDX-FileCopyrightText: Copyright 2024-2025 Arm Limited and/or its affiliates <open-source-office@arm.com>
3 //
4 // SPDX-License-Identifier: Apache-2.0
5 //
6
7 // Do not flag up inline assembly blocks
8 #pragma GCC diagnostic ignored "-Woverlength-strings"
9
10 #if !defined(__ARM_FEATURE_MATMUL_INT8)
11 #error "i8mm extension required to compile this micro-kernel"
12 #else
13 #include "kai_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_8x4x32_neon_i8mm.h"
14
15 #include <arm_neon.h>
16 #include <stddef.h>
17 #include <stdint.h>
18
19 #include "kai/kai_common.h"
20
21 static const size_t kai_m_step = 8;
22 static const size_t kai_n_step = 4;
23 static const size_t kai_mr = 4;
24 static const size_t kai_nr = 4;
25 static const size_t kai_kr = 16;
26 static const size_t kai_sr = 2;
27 static const size_t kai_bl = 32;
28 static const size_t kai_num_bytes_multiplier = sizeof(uint16_t);
29
30 46 inline static size_t kai_num_bytes_per_block_lhs(void) {
31 46 return kai_bl * sizeof(int8_t) + kai_num_bytes_multiplier;
32 }
33
34 46 inline static size_t kai_num_bytes_per_block_rhs(void) {
35 46 return (kai_bl / 2) * sizeof(int8_t) + kai_num_bytes_multiplier;
36 }
37
38 92 inline static size_t kai_num_blocks_per_row(size_t k) {
39 KAI_ASSUME((k % kai_bl) == 0);
40 92 return k / kai_bl;
41 }
42
43 46 inline static size_t kai_lhs_packed_stride(size_t k) {
44 46 return kai_mr * kai_num_blocks_per_row(k) * kai_num_bytes_per_block_lhs();
45 }
46
47 46 inline static size_t kai_rhs_packed_stride(size_t k) {
48 KAI_ASSUME((k % 2) == 0);
49 KAI_ASSUME((k % kai_kr) == 0);
50 KAI_ASSUME((k % kai_bl) == 0);
51
52 46 const size_t num_blocks_per_row = kai_num_blocks_per_row(k);
53 46 const size_t num_bytes_per_block = kai_num_bytes_per_block_rhs();
54
55 92 return kai_nr * (num_bytes_per_block * num_blocks_per_row);
56 46 }
57
58 72 size_t kai_get_m_step_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_8x4x32_neon_i8mm(void) {
59 72 return kai_m_step;
60 }
61
62 72 size_t kai_get_n_step_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_8x4x32_neon_i8mm(void) {
63 72 return kai_n_step;
64 }
65
66 48 size_t kai_get_mr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_8x4x32_neon_i8mm(void) {
67 48 return kai_mr;
68 }
69
70 48 size_t kai_get_nr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_8x4x32_neon_i8mm(void) {
71 48 return kai_nr;
72 }
73
74 72 size_t kai_get_kr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_8x4x32_neon_i8mm(void) {
75 72 return kai_kr;
76 }
77
78 48 size_t kai_get_sr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_8x4x32_neon_i8mm(void) {
79 48 return kai_sr;
80 }
81
82 46 size_t kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_8x4x32_neon_i8mm(
83 size_t m_idx, size_t k, size_t bl) {
84 KAI_ASSUME(bl == kai_bl);
85 KAI_ASSUME((k % 2) == 0);
86 KAI_ASSUME((k % kai_kr) == 0);
87 KAI_ASSUME((k % bl) == 0);
88 KAI_ASSUME((m_idx % kai_m_step) == 0);
89
90 46 return (m_idx / kai_mr) * kai_lhs_packed_stride(k);
91 }
92
93 46 size_t kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_8x4x32_neon_i8mm(
94 size_t n_idx, size_t k, size_t bl) {
95 KAI_ASSUME(bl == kai_bl);
96 KAI_ASSUME((k % 2) == 0);
97 KAI_ASSUME((k % kai_kr) == 0);
98 KAI_ASSUME((k % bl) == 0);
99 KAI_ASSUME((n_idx % kai_n_step) == 0);
100
101 46 return (n_idx / kai_nr) * kai_rhs_packed_stride(k);
102 }
103
104 23 size_t kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_8x4x32_neon_i8mm(
105 size_t m_idx, size_t n_idx, size_t dst_stride) {
106 KAI_ASSUME((m_idx % kai_m_step) == 0);
107 KAI_ASSUME((n_idx % kai_n_step) == 0);
108
109 23 return (n_idx * sizeof(float)) + m_idx * dst_stride;
110 }
111
112 23 size_t kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_8x4x32_neon_i8mm(size_t m, size_t n) {
113 23 return m * n * sizeof(float);
114 }
115
116 24 void kai_run_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_8x4x32_neon_i8mm(
117 size_t m, size_t n, size_t k, size_t bl, const void* lhs_packed, const void* rhs_packed,
118 float* dst, // NOLINT(readability-non-const-parameter)
119 size_t dst_stride_row, size_t dst_stride_col, float scalar_min, float scalar_max) {
120 KAI_ASSUME(bl == kai_bl);
121 KAI_ASSUME(k % kai_bl == 0);
122 KAI_ASSUME(dst_stride_col == sizeof(float));
123
124
1/2
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
24 if (m == 0) {
125 return;
126 }
127
128 24 const size_t num_blocks = k / kai_bl;
129 24 float clamp_vals[2] = {scalar_min, scalar_max};
130
131 48 __asm__ __volatile__(
132 "mov x12, %x[m]\n"
133 "mov x11, #0x88\n"
134 "movi v13.16b, #0xf0\n"
135 "cmp x12, #0x8\n"
136 "mul x11, %x[num_blocks], x11\n"
137 "blt 8f\n"
138 "1:" // Row loop
139 "mov x10, %x[rhs_packed]\n"
140 "mov x9, %x[n]\n"
141 "add x28, %x[dst], %x[dst_stride_row], LSL #3\n"
142 "2:" // Column loop
143 "mov x22, %x[lhs_packed]\n"
144 "movi v1.16b, #0x0\n"
145 "movi v22.16b, #0x0\n"
146 "mov x21, %x[num_blocks]\n"
147 "movi v14.16b, #0x0\n"
148 "movi v12.16b, #0x0\n"
149 "movi v15.16b, #0x0\n"
150 "movi v19.16b, #0x0\n"
151 "movi v3.16b, #0x0\n"
152 "movi v2.16b, #0x0\n"
153 "add x20, x22, x11\n"
154 "3:" // Block loop
155 "ldr d11, [x10, #0x0]\n"
156 "ldr d10, [x22, #0x0]\n"
157 "add x10, x10, #0x8\n"
158 "add x22, x22, #0x8\n"
159 "ldr q25, [x10, #0x0]\n"
160 "ldr q30, [x10, #0x10]\n"
161 "movi v6.4s, #0x0\n"
162 "movi v21.4s, #0x0\n"
163 "ldr d24, [x20, #0x0]\n"
164 "ldr q28, [x22, #0x0]\n"
165 "add x20, x20, #0x8\n"
166 "movi v9.4s, #0x0\n"
167 "ldr q4, [x22, #0x10]\n"
168 "ldr q23, [x20, #0x0]\n"
169 "movi v0.4s, #0x0\n"
170 "movi v31.4s, #0x0\n"
171 "ldr q17, [x20, #0x10]\n"
172 "ldr q18, [x10, #0x20]\n"
173 "shl v20.16b, v25.16b, #0x4\n"
174 "shl v29.16b, v30.16b, #0x4\n"
175 "ldr q16, [x10, #0x30]\n"
176 "ldr q26, [x22, #0x20]\n"
177 "movi v7.4s, #0x0\n"
178 "movi v27.4s, #0x0\n"
179 "ldr q8, [x22, #0x30]\n"
180 "ldr q5, [x20, #0x20]\n"
181 "and v25.16b, v25.16b, v13.16b\n"
182 "and v30.16b, v30.16b, v13.16b\n"
183 ".inst 0x4e94a786 // smmla v6.4s, v28.16b, v20.16b\n"
184 ".inst 0x4e9da795 // smmla v21.4s, v28.16b, v29.16b\n"
185 "ldr q28, [x20, #0x30]\n"
186 "fcvtl v11.4s, v11.4h\n"
187 ".inst 0x4e94a489 // smmla v9.4s, v4.16b, v20.16b\n"
188 ".inst 0x4e9da480 // smmla v0.4s, v4.16b, v29.16b\n"
189 "ldr q4, [x22, #0x40]\n"
190 "fcvtl v10.4s, v10.4h\n"
191 ".inst 0x4e94a6ff // smmla v31.4s, v23.16b, v20.16b\n"
192 ".inst 0x4e9da6e7 // smmla v7.4s, v23.16b, v29.16b\n"
193 "ldr q23, [x22, #0x50]\n"
194 "fcvtl v24.4s, v24.4h\n"
195 ".inst 0x4e94a63b // smmla v27.4s, v17.16b, v20.16b\n"
196 "movi v20.4s, #0x0\n"
197 "subs x21, x21, #0x1\n"
198 "add x10, x10, #0x40\n"
199 ".inst 0x4e9da634 // smmla v20.4s, v17.16b, v29.16b\n"
200 "ldr q17, [x20, #0x40]\n"
201 "shl v29.16b, v18.16b, #0x4\n"
202 "and v18.16b, v18.16b, v13.16b\n"
203 ".inst 0x4e9da746 // smmla v6.4s, v26.16b, v29.16b\n"
204 ".inst 0x4e9da509 // smmla v9.4s, v8.16b, v29.16b\n"
205 ".inst 0x4e9da4bf // smmla v31.4s, v5.16b, v29.16b\n"
206 ".inst 0x4e9da79b // smmla v27.4s, v28.16b, v29.16b\n"
207 "ldr q29, [x20, #0x50]\n"
208 ".inst 0x4e99a486 // smmla v6.4s, v4.16b, v25.16b\n"
209 ".inst 0x4e99a6e9 // smmla v9.4s, v23.16b, v25.16b\n"
210 ".inst 0x4e99a63f // smmla v31.4s, v17.16b, v25.16b\n"
211 ".inst 0x4e99a7bb // smmla v27.4s, v29.16b, v25.16b\n"
212 "shl v25.16b, v16.16b, #0x4\n"
213 "and v16.16b, v16.16b, v13.16b\n"
214 ".inst 0x4e99a755 // smmla v21.4s, v26.16b, v25.16b\n"
215 "ldr q26, [x22, #0x60]\n"
216 ".inst 0x4e99a500 // smmla v0.4s, v8.16b, v25.16b\n"
217 "ldr q8, [x22, #0x70]\n"
218 "add x22, x22, #0x80\n"
219 ".inst 0x4e99a4a7 // smmla v7.4s, v5.16b, v25.16b\n"
220 "ldr q5, [x20, #0x60]\n"
221 ".inst 0x4e99a794 // smmla v20.4s, v28.16b, v25.16b\n"
222 "ldr q25, [x20, #0x70]\n"
223 "fmul v28.4s, v11.4s, v10.s[0]\n"
224 "add x20, x20, #0x80\n"
225 ".inst 0x4e92a746 // smmla v6.4s, v26.16b, v18.16b\n"
226 ".inst 0x4e9ea495 // smmla v21.4s, v4.16b, v30.16b\n"
227 "fmul v4.4s, v11.4s, v10.s[1]\n"
228 ".inst 0x4e9ea6e0 // smmla v0.4s, v23.16b, v30.16b\n"
229 ".inst 0x4e92a509 // smmla v9.4s, v8.16b, v18.16b\n"
230 "fmul v23.4s, v11.4s, v10.s[2]\n"
231 ".inst 0x4e9ea627 // smmla v7.4s, v17.16b, v30.16b\n"
232 ".inst 0x4e92a4bf // smmla v31.4s, v5.16b, v18.16b\n"
233 "fmul v17.4s, v11.4s, v10.s[3]\n"
234 ".inst 0x4e9ea7b4 // smmla v20.4s, v29.16b, v30.16b\n"
235 ".inst 0x4e92a73b // smmla v27.4s, v25.16b, v18.16b\n"
236 "fmul v30.4s, v11.4s, v24.s[0]\n"
237 ".inst 0x4e90a755 // smmla v21.4s, v26.16b, v16.16b\n"
238 "fmul v29.4s, v11.4s, v24.s[1]\n"
239 ".inst 0x4e90a500 // smmla v0.4s, v8.16b, v16.16b\n"
240 "fmul v18.4s, v11.4s, v24.s[2]\n"
241 "fmul v10.4s, v11.4s, v24.s[3]\n"
242 ".inst 0x4e90a4a7 // smmla v7.4s, v5.16b, v16.16b\n"
243 ".inst 0x4e90a734 // smmla v20.4s, v25.16b, v16.16b\n"
244 "uzp1 v26.2d, v6.2d, v21.2d\n"
245 "uzp2 v6.2d, v6.2d, v21.2d\n"
246 "uzp1 v24.2d, v9.2d, v0.2d\n"
247 "uzp2 v16.2d, v9.2d, v0.2d\n"
248 "uzp1 v8.2d, v31.2d, v7.2d\n"
249 "uzp2 v11.2d, v31.2d, v7.2d\n"
250 "scvtf v26.4s, v26.4s, #0x4\n"
251 "uzp1 v31.2d, v27.2d, v20.2d\n"
252 "uzp2 v7.2d, v27.2d, v20.2d\n"
253 "scvtf v6.4s, v6.4s, #0x4\n"
254 "scvtf v24.4s, v24.4s, #0x4\n"
255 "scvtf v16.4s, v16.4s, #0x4\n"
256 "scvtf v8.4s, v8.4s, #0x4\n"
257 "fmla v1.4s, v26.4s, v28.4s\n"
258 "scvtf v11.4s, v11.4s, #0x4\n"
259 "scvtf v31.4s, v31.4s, #0x4\n"
260 "scvtf v7.4s, v7.4s, #0x4\n"
261 "fmla v22.4s, v6.4s, v4.4s\n"
262 "fmla v14.4s, v24.4s, v23.4s\n"
263 "fmla v12.4s, v16.4s, v17.4s\n"
264 "fmla v15.4s, v8.4s, v30.4s\n"
265 "fmla v19.4s, v11.4s, v29.4s\n"
266 "fmla v3.4s, v31.4s, v18.4s\n"
267 "fmla v2.4s, v7.4s, v10.4s\n"
268 "bgt 3b\n"
269 "ld1r { v17.4s }, [%x[clamp_vals]]\n"
270 "add x20, %x[clamp_vals], #0x4\n"
271 "cmp x9, #0x4\n"
272 "ld1r { v10.4s }, [x20]\n"
273 "fmax v1.4s, v1.4s, v17.4s\n"
274 "fmax v22.4s, v22.4s, v17.4s\n"
275 "fmax v14.4s, v14.4s, v17.4s\n"
276 "fmax v12.4s, v12.4s, v17.4s\n"
277 "fmax v15.4s, v15.4s, v17.4s\n"
278 "fmax v19.4s, v19.4s, v17.4s\n"
279 "fmax v3.4s, v3.4s, v17.4s\n"
280 "fmax v2.4s, v2.4s, v17.4s\n"
281 "fmin v1.4s, v1.4s, v10.4s\n"
282 "fmin v22.4s, v22.4s, v10.4s\n"
283 "fmin v14.4s, v14.4s, v10.4s\n"
284 "fmin v12.4s, v12.4s, v10.4s\n"
285 "fmin v15.4s, v15.4s, v10.4s\n"
286 "fmin v19.4s, v19.4s, v10.4s\n"
287 "fmin v3.4s, v3.4s, v10.4s\n"
288 "fmin v2.4s, v2.4s, v10.4s\n"
289 "blt 4f\n"
290 "mov x20, %x[dst]\n"
291 "str q1, [x20, #0x0]\n"
292 "add x20, x20, %x[dst_stride_row]\n"
293 "str q22, [x20, #0x0]\n"
294 "add x20, x20, %x[dst_stride_row]\n"
295 "str q14, [x20, #0x0]\n"
296 "add x20, x20, %x[dst_stride_row]\n"
297 "str q12, [x20, #0x0]\n"
298 "add x20, x20, %x[dst_stride_row]\n"
299 "str q15, [x20, #0x0]\n"
300 "add x20, x20, %x[dst_stride_row]\n"
301 "str q19, [x20, #0x0]\n"
302 "add x20, x20, %x[dst_stride_row]\n"
303 "str q3, [x20, #0x0]\n"
304 "add x20, x20, %x[dst_stride_row]\n"
305 "str q2, [x20, #0x0]\n"
306 "b 7f\n"
307 "4:" // Partial output
308 "mov x27, %x[dst]\n"
309 "add x26, x27, %x[dst_stride_row], LSL #2\n"
310 "add x25, x26, %x[dst_stride_row], LSL #1\n"
311 "add x24, x26, %x[dst_stride_row]\n"
312 "add x23, x25, %x[dst_stride_row]\n"
313 "add x22, x27, %x[dst_stride_row], LSL #1\n"
314 "add x21, x27, %x[dst_stride_row]\n"
315 "add x20, x22, %x[dst_stride_row]\n"
316 "tbz x9, #1, 5f\n"
317 "st1 { v2.d }[0], [x23], #0x8\n"
318 "st1 { v3.d }[0], [x25], #0x8\n"
319 "st1 { v19.d }[0], [x24], #0x8\n"
320 "st1 { v15.d }[0], [x26], #0x8\n"
321 "st1 { v12.d }[0], [x20], #0x8\n"
322 "st1 { v14.d }[0], [x22], #0x8\n"
323 "st1 { v22.d }[0], [x21], #0x8\n"
324 "st1 { v1.d }[0], [x27], #0x8\n"
325 "tbz x9, #0, 6f\n"
326 "st1 { v2.s }[2], [x23]\n"
327 "st1 { v3.s }[2], [x25]\n"
328 "st1 { v19.s }[2], [x24]\n"
329 "st1 { v15.s }[2], [x26]\n"
330 "st1 { v12.s }[2], [x20]\n"
331 "st1 { v14.s }[2], [x22]\n"
332 "st1 { v22.s }[2], [x21]\n"
333 "st1 { v1.s }[2], [x27]\n"
334 "b 6f\n"
335 "5:" // Output block 0: partial_1_0
336 "st1 { v2.s }[0], [x23]\n"
337 "st1 { v3.s }[0], [x25]\n"
338 "st1 { v19.s }[0], [x24]\n"
339 "st1 { v15.s }[0], [x26]\n"
340 "st1 { v12.s }[0], [x20]\n"
341 "st1 { v14.s }[0], [x22]\n"
342 "st1 { v22.s }[0], [x21]\n"
343 "st1 { v1.s }[0], [x27]\n"
344 "6:" // Output block 0: Done
345 "7:" // Output stage exit
346 "subs x9, x9, #0x4\n"
347 "add %x[dst], %x[dst], #0x10\n"
348 "bgt 2b\n"
349 "mov x20, #0x2\n"
350 "sub x12, x12, #0x8\n"
351 "cmp x12, #0x8\n"
352 "mov %x[dst], x28\n"
353 "madd %x[lhs_packed], x20, x11, %x[lhs_packed]\n"
354 "bge 1b\n"
355 "8:" // Row loop skip
356 "cbz x12, 16f\n"
357 "9:" // Row tail: Row loop
358 "mov x26, %x[rhs_packed]\n"
359 "mov x25, %x[n]\n"
360 "add x24, %x[dst], %x[dst_stride_row], LSL #2\n"
361 "10:" // Row tail: Column loop
362 "movi v1.16b, #0x0\n"
363 "movi v22.16b, #0x0\n"
364 "mov x22, %x[lhs_packed]\n"
365 "mov x20, %x[num_blocks]\n"
366 "movi v14.16b, #0x0\n"
367 "movi v12.16b, #0x0\n"
368 "11:" // Row tail: Block loop
369 "ldr d16, [x26, #0x0]\n"
370 "ldr d6, [x22, #0x0]\n"
371 "add x26, x26, #0x8\n"
372 "add x22, x22, #0x8\n"
373 "ldr q5, [x26, #0x0]\n"
374 "ldr q4, [x26, #0x10]\n"
375 "movi v7.4s, #0x0\n"
376 "movi v2.4s, #0x0\n"
377 "ldr q23, [x22, #0x0]\n"
378 "ldr q27, [x22, #0x10]\n"
379 "movi v0.4s, #0x0\n"
380 "movi v31.4s, #0x0\n"
381 "ldr q30, [x26, #0x20]\n"
382 "ldr q29, [x26, #0x30]\n"
383 "fcvtl v28.4s, v16.4h\n"
384 "fcvtl v6.4s, v6.4h\n"
385 "ldr q8, [x22, #0x20]\n"
386 "ldr q26, [x22, #0x30]\n"
387 "shl v21.16b, v5.16b, #0x4\n"
388 "shl v20.16b, v4.16b, #0x4\n"
389 "ldr q25, [x22, #0x40]\n"
390 "ldr q24, [x22, #0x50]\n"
391 "and v5.16b, v5.16b, v13.16b\n"
392 "and v4.16b, v4.16b, v13.16b\n"
393 "ldr q19, [x22, #0x60]\n"
394 "ldr q18, [x22, #0x70]\n"
395 "shl v17.16b, v30.16b, #0x4\n"
396 "shl v16.16b, v29.16b, #0x4\n"
397 ".inst 0x4e95a6e7 // smmla v7.4s, v23.16b, v21.16b\n"
398 ".inst 0x4e94a6e2 // smmla v2.4s, v23.16b, v20.16b\n"
399 "and v30.16b, v30.16b, v13.16b\n"
400 "subs x20, x20, #0x1\n"
401 ".inst 0x4e95a760 // smmla v0.4s, v27.16b, v21.16b\n"
402 ".inst 0x4e94a77f // smmla v31.4s, v27.16b, v20.16b\n"
403 "and v29.16b, v29.16b, v13.16b\n"
404 "add x26, x26, #0x40\n"
405 "fmul v23.4s, v28.4s, v6.s[0]\n"
406 "fmul v10.4s, v28.4s, v6.s[1]\n"
407 "add x22, x22, #0x80\n"
408 "fmul v21.4s, v28.4s, v6.s[2]\n"
409 "fmul v20.4s, v28.4s, v6.s[3]\n"
410 ".inst 0x4e91a507 // smmla v7.4s, v8.16b, v17.16b\n"
411 ".inst 0x4e90a502 // smmla v2.4s, v8.16b, v16.16b\n"
412 ".inst 0x4e91a740 // smmla v0.4s, v26.16b, v17.16b\n"
413 ".inst 0x4e90a75f // smmla v31.4s, v26.16b, v16.16b\n"
414 ".inst 0x4e85a727 // smmla v7.4s, v25.16b, v5.16b\n"
415 ".inst 0x4e84a722 // smmla v2.4s, v25.16b, v4.16b\n"
416 ".inst 0x4e85a700 // smmla v0.4s, v24.16b, v5.16b\n"
417 ".inst 0x4e84a71f // smmla v31.4s, v24.16b, v4.16b\n"
418 ".inst 0x4e9ea667 // smmla v7.4s, v19.16b, v30.16b\n"
419 ".inst 0x4e9da662 // smmla v2.4s, v19.16b, v29.16b\n"
420 ".inst 0x4e9ea640 // smmla v0.4s, v18.16b, v30.16b\n"
421 ".inst 0x4e9da65f // smmla v31.4s, v18.16b, v29.16b\n"
422 "uzp1 v19.2d, v7.2d, v2.2d\n"
423 "uzp2 v18.2d, v7.2d, v2.2d\n"
424 "scvtf v19.4s, v19.4s, #0x4\n"
425 "uzp1 v17.2d, v0.2d, v31.2d\n"
426 "uzp2 v16.2d, v0.2d, v31.2d\n"
427 "scvtf v18.4s, v18.4s, #0x4\n"
428 "fmla v1.4s, v19.4s, v23.4s\n"
429 "scvtf v17.4s, v17.4s, #0x4\n"
430 "scvtf v16.4s, v16.4s, #0x4\n"
431 "fmla v22.4s, v18.4s, v10.4s\n"
432 "fmla v14.4s, v17.4s, v21.4s\n"
433 "fmla v12.4s, v16.4s, v20.4s\n"
434 "bgt 11b\n"
435 "ld1r { v17.4s }, [%x[clamp_vals]]\n"
436 "add x20, %x[clamp_vals], #0x4\n"
437 "cmp x25, #0x4\n"
438 "ld1r { v16.4s }, [x20]\n"
439 "fmax v1.4s, v1.4s, v17.4s\n"
440 "fmax v22.4s, v22.4s, v17.4s\n"
441 "fmax v14.4s, v14.4s, v17.4s\n"
442 "fmax v12.4s, v12.4s, v17.4s\n"
443 "fmin v1.4s, v1.4s, v16.4s\n"
444 "fmin v22.4s, v22.4s, v16.4s\n"
445 "fmin v14.4s, v14.4s, v16.4s\n"
446 "fmin v12.4s, v12.4s, v16.4s\n"
447 "blt 12f\n"
448 "mov x20, %x[dst]\n"
449 "cmp x12, #0x1\n"
450 "str q1, [x20, #0x0]\n"
451 "add x20, x20, %x[dst_stride_row]\n"
452 "ble 15f\n"
453 "cmp x12, #0x2\n"
454 "str q22, [x20, #0x0]\n"
455 "add x20, x20, %x[dst_stride_row]\n"
456 "ble 15f\n"
457 "cmp x12, #0x3\n"
458 "str q14, [x20, #0x0]\n"
459 "add x20, x20, %x[dst_stride_row]\n"
460 "ble 15f\n"
461 "str q12, [x20, #0x0]\n"
462 "b 15f\n"
463 "12:" // Row tail: Partial output
464 "mov x23, %x[dst]\n"
465 "cmp x12, #0x1\n"
466 "add x22, x23, %x[dst_stride_row]\n"
467 "csel x22, x22, x23, GT\n"
468 "cmp x12, #0x2\n"
469 "add x21, x23, %x[dst_stride_row], LSL #1\n"
470 "csel x21, x21, x22, GT\n"
471 "cmp x12, #0x3\n"
472 "add x20, x21, %x[dst_stride_row]\n"
473 "csel x20, x20, x21, GT\n"
474 "tbz x25, #1, 13f\n"
475 "st1 { v12.d }[0], [x20], #0x8\n"
476 "st1 { v14.d }[0], [x21], #0x8\n"
477 "st1 { v22.d }[0], [x22], #0x8\n"
478 "st1 { v1.d }[0], [x23], #0x8\n"
479 "tbz x25, #0, 14f\n"
480 "st1 { v12.s }[2], [x20]\n"
481 "st1 { v14.s }[2], [x21]\n"
482 "st1 { v22.s }[2], [x22]\n"
483 "st1 { v1.s }[2], [x23]\n"
484 "b 14f\n"
485 "13:" // Row tail: Output block 0: partial_1_0
486 "st1 { v12.s }[0], [x20]\n"
487 "st1 { v14.s }[0], [x21]\n"
488 "st1 { v22.s }[0], [x22]\n"
489 "st1 { v1.s }[0], [x23]\n"
490 "14:" // Row tail: Output block 0: Done
491 "15:" // Row tail: Output stage exit
492 "subs x25, x25, #0x4\n"
493 "add %x[dst], %x[dst], #0x10\n"
494 "bgt 10b\n"
495 "subs x12, x12, #0x4\n"
496 "add %x[lhs_packed], %x[lhs_packed], x11\n"
497 "mov %x[dst], x24\n"
498 "bgt 9b\n"
499 "16:" // Row tail: Row loop skip
500 : [dst] "+&r"(dst), [lhs_packed] "+&r"(lhs_packed)
501 24 : [clamp_vals] "r"(clamp_vals), [dst_stride_row] "r"(dst_stride_row), [m] "r"(m), [n] "r"(n),
502 24 [num_blocks] "r"(num_blocks), [rhs_packed] "r"(rhs_packed)
503 : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
504 "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29",
505 "v30", "v31", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28");
506 24 }
507
508 #endif // Architectural feature check
509