Line | Branch | Exec | Source |
---|---|---|---|
1 | // | ||
2 | // SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its affiliates <open-source-office@arm.com> | ||
3 | // | ||
4 | // SPDX-License-Identifier: Apache-2.0 | ||
5 | // | ||
6 | #if !defined(__aarch64__) && !defined(__ARM_FEATURE_MATMUL_INT8) && !defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && \ | ||
7 | !defined(_M_ARM64) | ||
8 | #error "I8mm extension and fp16 vector arithmetic required to compile this micro-kernel" | ||
9 | #else // Architectural features check. | ||
10 | |||
11 | #include "kai_matmul_clamp_f16_qai8dxp4x8_qsi4cxp4x8_16x4_neon_i8mm.h" | ||
12 | |||
13 | #include <stddef.h> | ||
14 | #include <stdint.h> | ||
15 | |||
16 | #include "kai/kai_common.h" | ||
17 | |||
18 | typedef struct { | ||
19 | void* dst; | ||
20 | const void* lhs_packed; | ||
21 | const void* rhs_packed; | ||
22 | const float* clamp_vals; | ||
23 | size_t dst_stride_row; | ||
24 | size_t m; | ||
25 | size_t n; | ||
26 | size_t num_blocks; | ||
27 | } KernelArgs; | ||
28 | |||
29 | void kai_kernel_matmul_clamp_f16_qai8dxp4x8_qsi4cxp4x8_16x4_neon_i8mm(KernelArgs* args_ptr); | ||
30 | |||
31 | // Compute args | ||
32 | static const size_t kai_m_step = 16; | ||
33 | static const size_t kai_n_step = 4; | ||
34 | // Packing args | ||
35 | static const size_t kai_mr = 4; | ||
36 | static const size_t kai_nr = 4; | ||
37 | static const size_t kai_kr = 16; | ||
38 | static const size_t kai_sr = 2; | ||
39 | // LHS format args (num. bytes per value, multiplier, zero_point (if asymmetric)) | ||
40 | static const size_t kai_num_bytes_qvalue_lhs = 1; | ||
41 | static const size_t kai_num_bytes_multiplier_lhs = 4; | ||
42 | static const size_t kai_num_bytes_zp_lhs = 4; | ||
43 | // RHS format args (num. bytes per value, multiplier, zero_point (if asymmetric), and reduction sum (if LHS is | ||
44 | // asymmetric)) | ||
45 | static const size_t kai_num_bytes_recip_qvalue_rhs = 2; | ||
46 | static const size_t kai_num_bytes_multiplier_rhs = 4; | ||
47 | static const size_t kai_num_bytes_rsum_rhs = 4; | ||
48 | // DST format args | ||
49 | static const size_t kai_num_bytes_dst_value = 2; | ||
50 | // Extra args | ||
51 | static const size_t kai_num_bytes_bias = 4; | ||
52 | static const size_t kai_k_multiple_of = 32; | ||
53 | |||
54 | static const size_t kai_bl = 32; | ||
55 | |||
56 | 667 | inline static size_t kai_get_k_roundedup(size_t k) { | |
57 | 667 | return kai_roundup(k, kai_k_multiple_of); | |
58 | } | ||
59 | |||
60 | 222 | inline static size_t kai_get_lhs_packed_stride(size_t k) { | |
61 | 222 | const size_t k_internal = kai_get_k_roundedup(k); | |
62 | 222 | size_t lhs_packed_stride = kai_mr * ((k_internal * kai_num_bytes_qvalue_lhs) + kai_num_bytes_multiplier_lhs); | |
63 | // Since the LHS matrix is asymmetric with per-row quantization, we must include the | ||
64 | // the number of bytes to hold the zero point value | ||
65 | 222 | lhs_packed_stride += kai_mr * kai_num_bytes_zp_lhs; | |
66 | |||
67 | 444 | return lhs_packed_stride; | |
68 | 222 | } | |
69 | |||
70 | 222 | inline static size_t kai_get_rhs_packed_stride(size_t k) { | |
71 | 222 | const size_t k_internal = kai_get_k_roundedup(k); | |
72 | 222 | size_t rhs_packed_stride = kai_nr * (k_internal / kai_num_bytes_recip_qvalue_rhs); | |
73 | |||
74 | 222 | rhs_packed_stride += kai_nr * kai_num_bytes_multiplier_rhs; | |
75 | // Since the LHS matrix is quantized asymmetric with per-row quantization, we also include | ||
76 | // the number of bytes for the reduction sum | ||
77 | 222 | rhs_packed_stride += kai_nr * kai_num_bytes_rsum_rhs; | |
78 | // Since the bias is packed with the RHS matrix, the stride is adjusted with the number of bytes of the bias | ||
79 | 222 | rhs_packed_stride += kai_nr * kai_num_bytes_bias; | |
80 | |||
81 | 444 | return rhs_packed_stride; | |
82 | 222 | } | |
83 | |||
84 | 224 | size_t kai_get_m_step_matmul_clamp_f16_qai8dxp4x8_qsi4cxp4x8_16x4_neon_i8mm(void) { | |
85 | 224 | return kai_m_step; | |
86 | } | ||
87 | |||
88 | 224 | size_t kai_get_n_step_matmul_clamp_f16_qai8dxp4x8_qsi4cxp4x8_16x4_neon_i8mm(void) { | |
89 | 224 | return kai_n_step; | |
90 | } | ||
91 | |||
92 | 224 | size_t kai_get_mr_matmul_clamp_f16_qai8dxp4x8_qsi4cxp4x8_16x4_neon_i8mm(void) { | |
93 | 224 | return kai_mr; | |
94 | } | ||
95 | |||
96 | 224 | size_t kai_get_nr_matmul_clamp_f16_qai8dxp4x8_qsi4cxp4x8_16x4_neon_i8mm(void) { | |
97 | 224 | return kai_nr; | |
98 | } | ||
99 | |||
100 | 224 | size_t kai_get_kr_matmul_clamp_f16_qai8dxp4x8_qsi4cxp4x8_16x4_neon_i8mm(void) { | |
101 | 224 | return kai_kr; | |
102 | } | ||
103 | |||
104 | 224 | size_t kai_get_sr_matmul_clamp_f16_qai8dxp4x8_qsi4cxp4x8_16x4_neon_i8mm(void) { | |
105 | 224 | return kai_sr; | |
106 | } | ||
107 | |||
108 | 222 | size_t kai_get_lhs_packed_offset_matmul_clamp_f16_qai8dxp4x8_qsi4cxp4x8_16x4_neon_i8mm(size_t m_idx, size_t k) { | |
109 | − | KAI_ASSUME((m_idx % kai_m_step) == 0); | |
110 | |||
111 | 222 | return (m_idx / kai_mr) * kai_get_lhs_packed_stride(k); | |
112 | } | ||
113 | |||
114 | 222 | size_t kai_get_rhs_packed_offset_matmul_clamp_f16_qai8dxp4x8_qsi4cxp4x8_16x4_neon_i8mm(size_t n_idx, size_t k) { | |
115 | − | KAI_ASSUME((n_idx % kai_n_step) == 0); | |
116 | |||
117 | 222 | return (n_idx / kai_nr) * kai_get_rhs_packed_stride(k); | |
118 | } | ||
119 | |||
120 | 222 | size_t kai_get_dst_offset_matmul_clamp_f16_qai8dxp4x8_qsi4cxp4x8_16x4_neon_i8mm( | |
121 | size_t m_idx, size_t n_idx, size_t dst_stride) { | ||
122 | − | KAI_ASSUME((m_idx % kai_m_step) == 0); | |
123 | − | KAI_ASSUME((n_idx % kai_n_step) == 0); | |
124 | |||
125 | 222 | return (n_idx * kai_num_bytes_dst_value) + m_idx * dst_stride; | |
126 | } | ||
127 | |||
128 | 222 | size_t kai_get_dst_size_matmul_clamp_f16_qai8dxp4x8_qsi4cxp4x8_16x4_neon_i8mm(size_t m, size_t n) { | |
129 | 222 | return m * n * kai_num_bytes_dst_value; | |
130 | } | ||
131 | |||
132 | 223 | void kai_run_matmul_clamp_f16_qai8dxp4x8_qsi4cxp4x8_16x4_neon_i8mm( | |
133 | size_t m, // | ||
134 | size_t n, // | ||
135 | size_t k, // | ||
136 | const void* restrict lhs_packed, // | ||
137 | const void* restrict rhs_packed, // | ||
138 | void* restrict dst, // NOLINT(readability-non-const-parameter) | ||
139 | size_t dst_stride_row, // | ||
140 | size_t dst_stride_col, // | ||
141 | float scalar_min, // | ||
142 | float scalar_max) { | ||
143 | − | KAI_ASSUME(dst_stride_col == sizeof(uint16_t)); | |
144 |
1/2✓ Branch 0 taken 223 times.
✗ Branch 1 not taken.
|
223 | if (m == 0) { |
145 | ✗ | return; | |
146 | } | ||
147 | 223 | const size_t k_internal = kai_get_k_roundedup(k); | |
148 | 223 | size_t num_blocks = k_internal / kai_bl; | |
149 | 223 | const float clamp_vals[2] = {scalar_min, scalar_max}; | |
150 | |||
151 | 223 | KernelArgs args; | |
152 | |||
153 | 223 | args.dst = dst; | |
154 | 223 | args.lhs_packed = lhs_packed; | |
155 | 223 | args.rhs_packed = rhs_packed; | |
156 | 223 | args.clamp_vals = clamp_vals; | |
157 | 223 | args.dst_stride_row = dst_stride_row; | |
158 | 223 | args.m = m; | |
159 | 223 | args.n = n; | |
160 | 223 | args.num_blocks = num_blocks; | |
161 | |||
162 | 223 | kai_kernel_matmul_clamp_f16_qai8dxp4x8_qsi4cxp4x8_16x4_neon_i8mm(&args); | |
163 | 223 | } | |
164 | |||
165 | #endif // Architectural features check. | ||
166 |