KleidiAI Coverage Report

Directory: ./
Coverage: low: ≥ 0% medium: ≥ 75.0% high: ≥ 90.0%
Coverage Exec / Excl / Total
Lines: 100.0% 35 / 13 / 48
Functions: 100.0% 8 / 0 / 8
Branches: -% 0 / 26 / 26
kai/ukernels/matmul/pack/kai_rhs_pack_kxn_qsi8cxp2vlx4sb_qs8cx_f32_i32_sme.c
Line Branch Exec Source
1 //
2 // SPDX-FileCopyrightText: Copyright 2024-2025 Arm Limited and/or its affiliates <open-source-office@arm.com>
3 //
4 // SPDX-License-Identifier: Apache-2.0
5 //
6
7 #if (!defined(__aarch64__) || !defined(__ARM_FEATURE_SVE2)) && !defined(_M_ARM64)
8 #error This file must be compiled for AArch64, FEAT_SVE2.
9 #else // Architectural features check.
10
11 #include "kai_rhs_pack_kxn_qsi8cxp2vlx4sb_qs8cx_f32_i32_sme.h"
12
13 #include <stddef.h>
14 #include <stdint.h>
15 #include <string.h>
16
17 #include "kai/kai_common.h"
18
19 enum {
20 NR = 2,
21 KR = 4,
22 MAX_N_STEP = NR * ((KAI_SME_VEC_LENGTH_MAX_BYTES / sizeof(uint8_t)) / KR),
23 };
24
25 typedef struct {
26 const void* bias_ptr;
27 const void* scale_ptr;
28 int32_t input_zero_point;
29 float scale_multiplier;
30 size_t width;
31 size_t height;
32 size_t in_stride;
33 size_t out_stride;
34 const void* in;
35 void* out;
36 const void* pad_row;
37 } KernelArgs;
38
39 static const size_t kai_num_bytes_input = sizeof(uint8_t);
40 static const size_t kai_num_bytes_output = sizeof(uint8_t);
41 static const size_t kai_num_bytes_bias = sizeof(int32_t);
42 static const size_t kai_num_bytes_scale = sizeof(float);
43
44 void kai_kernel_rhs_pack_kxn_qsi8cxp2vlx4sb_qs8cx_f32_i32_sme(const KernelArgs* args_ptr);
45
46 9624 size_t kai_get_n_step_rhs_pack_kxn_qsi8cxp2vlx4sb_qs8cx_f32_i32_sme(void) {
47 9624 return NR * kai_get_sme_vector_length_u8() / KR;
48 }
49
50 834 size_t kai_get_rhs_offset_rhs_pack_kxn_qsi8cxp2vlx4sb_qs8cx_f32_i32_sme(size_t n_idx) {
51 − KAI_ASSUME(n_idx % kai_get_n_step_rhs_pack_kxn_qsi8cxp2vlx4sb_qs8cx_f32_i32_sme() == 0);
52
53 834 return n_idx * kai_num_bytes_input;
54 }
55
56 834 size_t kai_get_bias_offset_rhs_pack_kxn_qsi8cxp2vlx4sb_qs8cx_f32_i32_sme(size_t n_idx) {
57 834 return n_idx * kai_num_bytes_bias;
58 }
59
60 834 size_t kai_get_scale_offset_rhs_pack_kxn_qsi8cxp2vlx4sb_qs8cx_f32_i32_sme(size_t n_idx) {
61 834 return n_idx * kai_num_bytes_scale;
62 }
63
64 2652 size_t kai_get_rhs_packed_stride_rhs_pack_kxn_qsi8cxp2vlx4sb_qs8cx_f32_i32_sme(size_t k) {
65 5304 return kai_get_n_step_rhs_pack_kxn_qsi8cxp2vlx4sb_qs8cx_f32_i32_sme() *
66 2652 (kai_num_bytes_bias + kai_roundup(k, KR) * kai_num_bytes_output + kai_num_bytes_scale);
67 }
68
69 1818 size_t kai_get_rhs_packed_offset_rhs_pack_kxn_qsi8cxp2vlx4sb_qs8cx_f32_i32_sme(size_t n_idx, size_t k) {
70 − KAI_ASSUME(n_idx % kai_get_n_step_rhs_pack_kxn_qsi8cxp2vlx4sb_qs8cx_f32_i32_sme() == 0);
71
72 1818 const size_t block_idx = n_idx / kai_get_n_step_rhs_pack_kxn_qsi8cxp2vlx4sb_qs8cx_f32_i32_sme();
73 3636 return block_idx * kai_get_rhs_packed_stride_rhs_pack_kxn_qsi8cxp2vlx4sb_qs8cx_f32_i32_sme(k);
74 1818 }
75
76 834 size_t kai_get_rhs_packed_size_rhs_pack_kxn_qsi8cxp2vlx4sb_qs8cx_f32_i32_sme(size_t n, size_t k) {
77 834 const size_t n_nr_blocks = kai_roundup(n, kai_get_n_step_rhs_pack_kxn_qsi8cxp2vlx4sb_qs8cx_f32_i32_sme());
78 1668 return kai_get_rhs_packed_offset_rhs_pack_kxn_qsi8cxp2vlx4sb_qs8cx_f32_i32_sme(n_nr_blocks, k);
79 834 }
80
81 834 void kai_run_rhs_pack_kxn_qsi8cxp2vlx4sb_qs8cx_f32_i32_sme(
82 size_t num_groups, size_t n, size_t k, size_t nr, size_t kr, size_t sr, size_t rhs_stride_row, const void* rhs,
83 const void* bias, const void* scale, void* rhs_packed, size_t extra_bytes,
84 const struct kai_rhs_pack_qsi8cx_params* params) {
85 − KAI_ASSUME(num_groups == 1);
86 − KAI_ASSUME(nr == kai_get_n_step_rhs_pack_kxn_qsi8cxp2vlx4sb_qs8cx_f32_i32_sme());
87 − KAI_ASSUME(kr == KR);
88 − KAI_ASSUME(sr == 1);
89 − KAI_ASSUME(rhs != NULL);
90 − KAI_ASSUME(bias != NULL);
91 − KAI_ASSUME(scale != NULL);
92 − KAI_ASSUME(rhs_packed != NULL);
93 − KAI_ASSUME(extra_bytes == 0);
94 − KAI_ASSUME(params != NULL);
95
96 − KAI_ASSERT(kai_get_n_step_rhs_pack_kxn_qsi8cxp2vlx4sb_qs8cx_f32_i32_sme() <= MAX_N_STEP);
97 static const uint8_t pad_row[MAX_N_STEP] = {0};
98
99 834 KernelArgs args;
100 834 args.bias_ptr = bias;
101 834 args.scale_ptr = scale;
102 834 args.height = k;
103 834 args.width = n;
104 834 args.in = rhs;
105 834 args.out = rhs_packed;
106 834 args.in_stride = rhs_stride_row;
107 834 args.out_stride = kai_get_rhs_packed_stride_rhs_pack_kxn_qsi8cxp2vlx4sb_qs8cx_f32_i32_sme(args.height);
108 834 args.input_zero_point = params->lhs_zero_point;
109 834 args.scale_multiplier = params->scale_multiplier;
110 834 args.pad_row = pad_row;
111
112 834 kai_commit_za();
113
114 834 kai_kernel_rhs_pack_kxn_qsi8cxp2vlx4sb_qs8cx_f32_i32_sme(&args);
115 834 }
116
117 #endif // Architectural features check.
118