KleidiAI Coverage Report

Directory: ./
Coverage: low: ≥ 0% medium: ≥ 75.0% high: ≥ 90.0%
Coverage Exec / Excl / Total
Lines: 98.1% 52 / 13 / 66
Functions: 100.0% 16 / 0 / 16
Branches: 50.0% 1 / 26 / 28
kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm.c
Line Branch Exec Source
1 //
2 // SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its affiliates <open-source-office@arm.com>
3 //
4 // SPDX-License-Identifier: Apache-2.0
5 //
6 #if (!defined(__aarch64__) || !defined(__ARM_FEATURE_SVE)) && !defined(_M_ARM64)
7 #error "This file must be compiled for AArch64, FEAT_SVE."
8 #elif !defined(__ARM_FEATURE_MATMUL_INT8) && !defined(_M_ARM64)
9 #error "I8mm extension required to compile this micro-kernel."
10 #else // Architectural features check.
11
12 #include "kai_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm.h"
13
14 #include <stddef.h>
15 #include <stdint.h>
16
17 #include "kai/kai_common.h"
18
19 typedef struct {
20 float* dst;
21 const void* lhs_packed;
22 const void* rhs_packed;
23 const float* clamp_vals;
24 size_t dst_stride_row;
25 size_t m;
26 size_t n;
27 size_t num_blocks;
28 } KernelArgs;
29
30 void kai_kernel_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm(KernelArgs* args_ptr);
31
32 // Compute args
33 static const size_t kai_m_step = 16;
34 static const size_t kai_n_step = 8;
35 // Packing args
36 static const size_t kai_mr = 4;
37 static const size_t kai_nr = 8;
38 static const size_t kai_kr = 16;
39 static const size_t kai_sr = 2;
40 // LHS format args (num. bytes per value, multiplier, zero_point (if asymmetric))
41 static const size_t kai_num_bytes_qvalue_lhs = 1;
42 static const size_t kai_num_bytes_multiplier_lhs = 2;
43 // RHS format args (num. bytes per value, multiplier, zero_point (if asymmetric), and reduction sum (if LHS is
44 // asymmetric))
45 static const size_t kai_num_bytes_recip_qvalue_rhs = 2;
46 static const size_t kai_num_bytes_multiplier_rhs = 2;
47 // DST format args
48 static const size_t kai_num_bytes_dst_value = 4;
49 // Extra args
50 static const size_t kai_bl = 32;
51
52 184 inline static size_t kai_get_num_bytes_per_block_lhs(size_t bl) {
53 184 return (bl * kai_num_bytes_qvalue_lhs) + kai_num_bytes_multiplier_lhs;
54 }
55
56 184 inline static size_t kai_get_num_bytes_per_block_rhs(size_t bl) {
57 − KAI_ASSUME(bl == kai_bl);
58 184 size_t num_bytes_per_block_rhs = (bl / kai_num_bytes_recip_qvalue_rhs) + kai_num_bytes_multiplier_rhs;
59 368 return num_bytes_per_block_rhs;
60 184 }
61
62 462 inline static size_t kai_get_num_blocks_per_row(size_t k, size_t bl) {
63 − KAI_ASSUME(bl == kai_bl);
64 − KAI_ASSUME((k % kai_bl) == 0);
65
66 462 return kai_roundup(k, bl) / bl;
67 }
68
69 184 inline static size_t kai_get_lhs_packed_stride(size_t k, size_t bl) {
70 184 return kai_mr * kai_get_num_blocks_per_row(k, bl) * kai_get_num_bytes_per_block_lhs(bl);
71 }
72
73 184 inline static size_t kai_get_rhs_packed_stride(size_t k, size_t bl) {
74 − KAI_ASSUME(bl == kai_bl);
75 − KAI_ASSUME((k % kai_bl) == 0);
76
77 184 const size_t num_blocks_per_row = kai_get_num_blocks_per_row(k, bl);
78 184 const size_t num_bytes_per_block = kai_get_num_bytes_per_block_rhs(bl);
79
80 184 const size_t rhs_packed_stride = kai_nr * (num_bytes_per_block * num_blocks_per_row);
81
82 368 return rhs_packed_stride;
83 184 }
84
85 276 size_t kai_get_m_step_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm(void) {
86 276 return kai_m_step;
87 }
88
89 276 size_t kai_get_n_step_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm(void) {
90 276 return kai_n_step;
91 }
92
93 184 size_t kai_get_mr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm(void) {
94 184 return kai_mr;
95 }
96
97 184 size_t kai_get_nr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm(void) {
98 184 return kai_nr;
99 }
100
101 276 size_t kai_get_kr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm(void) {
102 276 return kai_kr;
103 }
104
105 184 size_t kai_get_sr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm(void) {
106 184 return kai_sr;
107 }
108
109 184 size_t kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm(
110 size_t m_idx, size_t k, size_t bl) {
111 − KAI_ASSUME((m_idx % kai_m_step) == 0);
112
113 184 return (m_idx / kai_mr) * kai_get_lhs_packed_stride(k, bl);
114 }
115
116 184 size_t kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm(
117 size_t n_idx, size_t k, size_t bl) {
118 − KAI_ASSUME((k % bl) == 0);
119 − KAI_ASSUME((n_idx % kai_n_step) == 0);
120
121 184 return (n_idx / kai_nr) * kai_get_rhs_packed_stride(k, bl);
122 }
123
124 92 size_t kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm(
125 size_t m_idx, size_t n_idx, size_t dst_stride) {
126 − KAI_ASSUME((m_idx % kai_m_step) == 0);
127 − KAI_ASSUME((n_idx % kai_n_step) == 0);
128
129 92 return (n_idx * kai_num_bytes_dst_value) + m_idx * dst_stride;
130 }
131
132 92 size_t kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm(size_t m, size_t n) {
133 92 return m * n * kai_num_bytes_dst_value;
134 }
135
136 94 void kai_run_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm(
137 size_t m, //
138 size_t n, //
139 size_t k, //
140 size_t bl, //
141 const void* restrict lhs_packed, //
142 const void* restrict rhs_packed, //
143 float* restrict dst, // NOLINT(readability-non-const-parameter)
144 size_t dst_stride_row, //
145 size_t dst_stride_col, //
146 float scalar_min, //
147 float scalar_max) {
148 − KAI_ASSUME((k % bl) == 0);
149 − KAI_ASSUME((bl % kai_bl) == 0);
150 94 KAI_UNUSED(dst_stride_col);
151
152 − KAI_ASSERT_MSG(
153 kai_get_sve_vector_length_u8() == 32, "Currently, this kernel only supports 256 bit vector lengths.");
154
1/2
✓ Branch 0 taken 94 times.
✗ Branch 1 not taken.
 94 if (m == 0) {
155 ✗ return;
156 }
157 94 const size_t num_blocks = kai_get_num_blocks_per_row(k, bl);
158 94 const float clamp_vals[2] = {scalar_min, scalar_max};
159
160 94 KernelArgs args;
161
162 94 args.dst = dst;
163 94 args.lhs_packed = lhs_packed;
164 94 args.rhs_packed = rhs_packed;
165 94 args.clamp_vals = clamp_vals;
166 94 args.dst_stride_row = dst_stride_row;
167 94 args.m = m;
168 94 args.n = n;
169 94 args.num_blocks = num_blocks;
170
171 94 kai_kernel_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm(&args);
172 94 }
173
174 #endif // Architectural features check.
175