Line |
Branch |
Exec |
Source |
1 |
|
|
// |
2 |
|
|
// SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its affiliates <open-source-office@arm.com> |
3 |
|
|
// |
4 |
|
|
// SPDX-License-Identifier: Apache-2.0 |
5 |
|
|
// |
6 |
|
|
|
7 |
|
|
#if (!defined(__aarch64__) || !defined(__ARM_FEATURE_SVE2)) && !defined(_M_ARM64) |
8 |
|
|
#error This file must be compiled for AArch64, FEAT_SVE2. |
9 |
|
|
#else // Architectural features check. |
10 |
|
|
|
11 |
|
|
#include "kai_matmul_clamp_f16_f16_f16p2vlx2b_1x8vl_sme_mla.h" |
12 |
|
|
|
13 |
|
|
#include <stddef.h> |
14 |
|
|
#include <stdint.h> |
15 |
|
|
|
16 |
|
|
#include "kai/kai_common.h" |
17 |
|
|
|
18 |
|
|
typedef struct { |
19 |
|
|
uint16_t maxval; |
20 |
|
|
uint16_t minval; |
21 |
|
|
const void* A_ptr; |
22 |
|
|
const void* B_ptr; |
23 |
|
|
size_t N; |
24 |
|
|
size_t K; |
25 |
|
|
void* output_ptr; |
26 |
|
|
uint64_t flags; |
27 |
|
|
} KernelArgs; |
28 |
|
|
|
29 |
|
|
static const size_t kai_m_step = 1; |
30 |
|
|
static const size_t kai_nr = 2; |
31 |
|
|
static const size_t kai_n_step = 8; |
32 |
|
|
static const size_t kai_kr = 2; |
33 |
|
|
static const size_t kai_sr = 1; |
34 |
|
|
|
35 |
|
|
void kai_kernel_matmul_clamp_f16_f16_f16p2vlx2b_1x8vl_sme_mla(KernelArgs* args_ptr); |
36 |
|
|
|
37 |
|
|
uint16_t kai_f16_from_float_matmul_clamp_f16_f16_f16p2vlx2b_1x8vl_sme_mla(float value); |
38 |
|
|
|
39 |
|
52 |
size_t kai_get_m_step_matmul_clamp_f16_f16_f16p2vlx2b_1x8vl_sme_mla(void) { |
40 |
|
52 |
return kai_m_step; |
41 |
|
|
} |
42 |
|
|
|
43 |
|
236 |
size_t kai_get_n_step_matmul_clamp_f16_f16_f16p2vlx2b_1x8vl_sme_mla(void) { |
44 |
|
236 |
return kai_n_step * kai_get_sme_vector_length_u16() / kai_kr; |
45 |
|
|
} |
46 |
|
|
|
47 |
|
46 |
size_t kai_get_nr_matmul_clamp_f16_f16_f16p2vlx2b_1x8vl_sme_mla(void) { |
48 |
|
46 |
return kai_nr * kai_get_sme_vector_length_u16() / kai_kr; |
49 |
|
|
} |
50 |
|
|
|
51 |
|
46 |
size_t kai_get_kr_matmul_clamp_f16_f16_f16p2vlx2b_1x8vl_sme_mla(void) { |
52 |
|
46 |
return kai_kr; |
53 |
|
|
} |
54 |
|
|
|
55 |
|
46 |
size_t kai_get_sr_matmul_clamp_f16_f16_f16p2vlx2b_1x8vl_sme_mla(void) { |
56 |
|
46 |
return kai_sr; |
57 |
|
|
} |
58 |
|
|
|
59 |
|
✗ |
size_t kai_get_lhs_offset_matmul_clamp_f16_f16_f16p2vlx2b_1x8vl_sme_mla(size_t m_idx, size_t k) { |
60 |
|
− |
KAI_ASSUME(m_idx == 0); |
61 |
|
|
|
62 |
|
✗ |
return m_idx * k; |
63 |
|
|
} |
64 |
|
|
|
65 |
|
46 |
static size_t kai_get_rhs_packed_stride_matmul_clamp_f16_f16_f16p2vlx2b_1x8vl_sme_mla(size_t k) { |
66 |
|
92 |
return kai_get_n_step_matmul_clamp_f16_f16_f16p2vlx2b_1x8vl_sme_mla() * |
67 |
|
46 |
(kai_roundup(k, kai_kr) * sizeof(uint16_t) + sizeof(uint16_t)); |
68 |
|
|
} |
69 |
|
|
|
70 |
|
46 |
size_t kai_get_rhs_packed_offset_matmul_clamp_f16_f16_f16p2vlx2b_1x8vl_sme_mla(size_t n_idx, size_t k) { |
71 |
|
− |
KAI_ASSUME(n_idx % kai_get_n_step_matmul_clamp_f16_f16_f16p2vlx2b_1x8vl_sme_mla() == 0); |
72 |
|
|
|
73 |
|
46 |
const size_t block_idx = n_idx / kai_get_n_step_matmul_clamp_f16_f16_f16p2vlx2b_1x8vl_sme_mla(); |
74 |
|
92 |
return block_idx * kai_get_rhs_packed_stride_matmul_clamp_f16_f16_f16p2vlx2b_1x8vl_sme_mla(k); |
75 |
|
46 |
} |
76 |
|
|
|
77 |
|
46 |
size_t kai_get_dst_offset_matmul_clamp_f16_f16_f16p2vlx2b_1x8vl_sme_mla(size_t m_idx, size_t n_idx, size_t dst_stride) { |
78 |
|
− |
KAI_ASSUME(m_idx == 0); |
79 |
|
− |
KAI_ASSUME(n_idx % kai_get_n_step_matmul_clamp_f16_f16_f16p2vlx2b_1x8vl_sme_mla() == 0); |
80 |
|
|
|
81 |
|
46 |
return (m_idx * dst_stride) + (n_idx * sizeof(uint16_t)); |
82 |
|
|
} |
83 |
|
|
|
84 |
|
46 |
size_t kai_get_dst_size_matmul_clamp_f16_f16_f16p2vlx2b_1x8vl_sme_mla(size_t m, size_t n) { |
85 |
|
46 |
return m * n * sizeof(uint16_t); |
86 |
|
|
} |
87 |
|
|
|
88 |
|
47 |
void kai_run_matmul_clamp_f16_f16_f16p2vlx2b_1x8vl_sme_mla( |
89 |
|
|
size_t m, size_t n, size_t k, const void* lhs, size_t lhs_stride, const void* rhs_packed, void* dst, |
90 |
|
|
size_t dst_stride_row, size_t dst_stride_col, float clamp_min, float clamp_max) { |
91 |
|
47 |
KAI_UNUSED(dst_stride_row); |
92 |
|
47 |
KAI_UNUSED(dst_stride_col); |
93 |
|
47 |
KAI_UNUSED(lhs_stride); |
94 |
|
− |
KAI_ASSUME(m == 1); |
95 |
|
|
|
96 |
|
47 |
uint64_t flags = 2; |
97 |
|
|
|
98 |
|
47 |
KernelArgs args; |
99 |
|
|
|
100 |
|
47 |
args.minval = kai_f16_from_float_matmul_clamp_f16_f16_f16p2vlx2b_1x8vl_sme_mla(clamp_min); |
101 |
|
47 |
args.maxval = kai_f16_from_float_matmul_clamp_f16_f16_f16p2vlx2b_1x8vl_sme_mla(clamp_max); |
102 |
|
47 |
args.A_ptr = lhs; |
103 |
|
47 |
args.B_ptr = rhs_packed; |
104 |
|
47 |
args.N = n; |
105 |
|
47 |
args.K = k; |
106 |
|
47 |
args.output_ptr = dst; |
107 |
|
47 |
args.flags = flags; |
108 |
|
|
|
109 |
|
47 |
kai_kernel_matmul_clamp_f16_f16_f16p2vlx2b_1x8vl_sme_mla(&args); |
110 |
|
47 |
} |
111 |
|
|
|
112 |
|
|
#endif // Architectural features check. |
113 |
|
|
|