Line | Branch | Exec | Source |
---|---|---|---|
1 | // | ||
2 | // SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its affiliates <open-source-office@arm.com> | ||
3 | // | ||
4 | // SPDX-License-Identifier: Apache-2.0 | ||
5 | // | ||
6 | #if ( \ | ||
7 | !defined(__aarch64__) || !defined(__ARM_FEATURE_MATMUL_INT8) || !defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC)) && \ | ||
8 | !defined(_M_ARM64) | ||
9 | #error "I8mm extension and bf16 vector arithmetic required to compile this micro-kernel" | ||
10 | #else // Architectural features check. | ||
11 | |||
12 | #include "kai_matmul_clamp_bf16_qai8dxp4x8_qsi4cxp8x8_8x8_neon_i8mm.h" | ||
13 | |||
14 | #include <stddef.h> | ||
15 | #include <stdint.h> | ||
16 | |||
17 | #include "kai/kai_common.h" | ||
18 | |||
19 | typedef struct { | ||
20 | void* dst; | ||
21 | const void* lhs_packed; | ||
22 | const void* rhs_packed; | ||
23 | const float* clamp_vals; | ||
24 | size_t dst_stride_row; | ||
25 | size_t m; | ||
26 | size_t n; | ||
27 | size_t num_blocks; | ||
28 | } KernelArgs; | ||
29 | |||
30 | void kai_kernel_matmul_clamp_bf16_qai8dxp4x8_qsi4cxp8x8_8x8_neon_i8mm(KernelArgs* args_ptr); | ||
31 | |||
32 | // Compute args | ||
33 | static const size_t kai_m_step = 8; | ||
34 | static const size_t kai_n_step = 8; | ||
35 | // Packing args | ||
36 | static const size_t kai_mr = 4; | ||
37 | static const size_t kai_nr = 8; | ||
38 | static const size_t kai_kr = 16; | ||
39 | static const size_t kai_sr = 2; | ||
40 | // LHS format args (num. bytes per value, multiplier, zero_point (if asymmetric)) | ||
41 | static const size_t kai_num_bytes_qvalue_lhs = 1; | ||
42 | static const size_t kai_num_bytes_multiplier_lhs = 4; | ||
43 | static const size_t kai_num_bytes_zp_lhs = 4; | ||
44 | // RHS format args (num. bytes per value, multiplier, zero_point (if asymmetric), and reduction sum (if LHS is | ||
45 | // asymmetric)) | ||
46 | static const size_t kai_num_bytes_recip_qvalue_rhs = 2; | ||
47 | static const size_t kai_num_bytes_multiplier_rhs = 4; | ||
48 | static const size_t kai_num_bytes_rsum_rhs = 4; | ||
49 | // DST format args | ||
50 | static const size_t kai_num_bytes_dst_value = 2; | ||
51 | // Extra args | ||
52 | static const size_t kai_num_bytes_bias = 4; | ||
53 | static const size_t kai_k_multiple_of = 32; | ||
54 | |||
55 | static const size_t kai_bl = 32; | ||
56 | |||
57 | 1165 | inline static size_t kai_get_k_roundedup(size_t k) { | |
58 | 1165 | return kai_roundup(k, kai_k_multiple_of); | |
59 | } | ||
60 | |||
61 | 388 | inline static size_t kai_get_lhs_packed_stride(size_t k) { | |
62 | 388 | const size_t k_internal = kai_get_k_roundedup(k); | |
63 | 388 | size_t lhs_packed_stride = kai_mr * ((k_internal * kai_num_bytes_qvalue_lhs) + kai_num_bytes_multiplier_lhs); | |
64 | // Since the LHS matrix is asymmetric with per-row quantization, we must include the | ||
65 | // the number of bytes to hold the zero point value | ||
66 | 388 | lhs_packed_stride += kai_mr * kai_num_bytes_zp_lhs; | |
67 | |||
68 | 776 | return lhs_packed_stride; | |
69 | 388 | } | |
70 | |||
71 | 388 | inline static size_t kai_get_rhs_packed_stride(size_t k) { | |
72 | 388 | const size_t k_internal = kai_get_k_roundedup(k); | |
73 | 388 | size_t rhs_packed_stride = kai_nr * (k_internal / kai_num_bytes_recip_qvalue_rhs); | |
74 | |||
75 | 388 | rhs_packed_stride += kai_nr * kai_num_bytes_multiplier_rhs; | |
76 | // Since the LHS matrix is quantized asymmetric with per-row quantization, we also include | ||
77 | // the number of bytes for the reduction sum | ||
78 | 388 | rhs_packed_stride += kai_nr * kai_num_bytes_rsum_rhs; | |
79 | // Since the bias is packed with the RHS matrix, the stride is adjusted with the number of bytes of the bias | ||
80 | 388 | rhs_packed_stride += kai_nr * kai_num_bytes_bias; | |
81 | |||
82 | 776 | return rhs_packed_stride; | |
83 | 388 | } | |
84 | |||
85 | 392 | size_t kai_get_m_step_matmul_clamp_bf16_qai8dxp4x8_qsi4cxp8x8_8x8_neon_i8mm(void) { | |
86 | 392 | return kai_m_step; | |
87 | } | ||
88 | |||
89 | 392 | size_t kai_get_n_step_matmul_clamp_bf16_qai8dxp4x8_qsi4cxp8x8_8x8_neon_i8mm(void) { | |
90 | 392 | return kai_n_step; | |
91 | } | ||
92 | |||
93 | 392 | size_t kai_get_mr_matmul_clamp_bf16_qai8dxp4x8_qsi4cxp8x8_8x8_neon_i8mm(void) { | |
94 | 392 | return kai_mr; | |
95 | } | ||
96 | |||
97 | 392 | size_t kai_get_nr_matmul_clamp_bf16_qai8dxp4x8_qsi4cxp8x8_8x8_neon_i8mm(void) { | |
98 | 392 | return kai_nr; | |
99 | } | ||
100 | |||
101 | 392 | size_t kai_get_kr_matmul_clamp_bf16_qai8dxp4x8_qsi4cxp8x8_8x8_neon_i8mm(void) { | |
102 | 392 | return kai_kr; | |
103 | } | ||
104 | |||
105 | 392 | size_t kai_get_sr_matmul_clamp_bf16_qai8dxp4x8_qsi4cxp8x8_8x8_neon_i8mm(void) { | |
106 | 392 | return kai_sr; | |
107 | } | ||
108 | |||
109 | 388 | size_t kai_get_lhs_packed_offset_matmul_clamp_bf16_qai8dxp4x8_qsi4cxp8x8_8x8_neon_i8mm(size_t m_idx, size_t k) { | |
110 | − | KAI_ASSUME((m_idx % kai_m_step) == 0); | |
111 | |||
112 | 388 | return (m_idx / kai_mr) * kai_get_lhs_packed_stride(k); | |
113 | } | ||
114 | |||
115 | 388 | size_t kai_get_rhs_packed_offset_matmul_clamp_bf16_qai8dxp4x8_qsi4cxp8x8_8x8_neon_i8mm(size_t n_idx, size_t k) { | |
116 | − | KAI_ASSUME((n_idx % kai_n_step) == 0); | |
117 | |||
118 | 388 | return (n_idx / kai_nr) * kai_get_rhs_packed_stride(k); | |
119 | } | ||
120 | |||
121 | 388 | size_t kai_get_dst_offset_matmul_clamp_bf16_qai8dxp4x8_qsi4cxp8x8_8x8_neon_i8mm( | |
122 | size_t m_idx, size_t n_idx, size_t dst_stride) { | ||
123 | − | KAI_ASSUME((m_idx % kai_m_step) == 0); | |
124 | − | KAI_ASSUME((n_idx % kai_n_step) == 0); | |
125 | |||
126 | 388 | return (n_idx * kai_num_bytes_dst_value) + m_idx * dst_stride; | |
127 | } | ||
128 | |||
129 | 388 | size_t kai_get_dst_size_matmul_clamp_bf16_qai8dxp4x8_qsi4cxp8x8_8x8_neon_i8mm(size_t m, size_t n) { | |
130 | 388 | return m * n * kai_num_bytes_dst_value; | |
131 | } | ||
132 | |||
133 | 389 | void kai_run_matmul_clamp_bf16_qai8dxp4x8_qsi4cxp8x8_8x8_neon_i8mm( | |
134 | size_t m, // | ||
135 | size_t n, // | ||
136 | size_t k, // | ||
137 | const void* restrict lhs_packed, // | ||
138 | const void* restrict rhs_packed, // | ||
139 | void* restrict dst, // NOLINT(readability-non-const-parameter) | ||
140 | size_t dst_stride_row, // | ||
141 | size_t dst_stride_col, // | ||
142 | float scalar_min, // | ||
143 | float scalar_max) { | ||
144 | − | KAI_ASSUME(dst_stride_col == sizeof(uint16_t)); | |
145 |
1/2✓ Branch 0 taken 389 times.
✗ Branch 1 not taken.
|
389 | if (m == 0) { |
146 | ✗ | return; | |
147 | } | ||
148 | 389 | const size_t k_internal = kai_get_k_roundedup(k); | |
149 | 389 | size_t num_blocks = k_internal / kai_bl; | |
150 | 389 | const float clamp_vals[2] = {scalar_min, scalar_max}; | |
151 | |||
152 | 389 | KernelArgs args; | |
153 | |||
154 | 389 | args.dst = dst; | |
155 | 389 | args.lhs_packed = lhs_packed; | |
156 | 389 | args.rhs_packed = rhs_packed; | |
157 | 389 | args.clamp_vals = clamp_vals; | |
158 | 389 | args.dst_stride_row = dst_stride_row; | |
159 | 389 | args.m = m; | |
160 | 389 | args.n = n; | |
161 | 389 | args.num_blocks = num_blocks; | |
162 | |||
163 | 389 | kai_kernel_matmul_clamp_bf16_qai8dxp4x8_qsi4cxp8x8_8x8_neon_i8mm(&args); | |
164 | 389 | } | |
165 | |||
166 | #endif // Architectural features check. | ||
167 |