KleidiAI Coverage Report

Directory: ./
Coverage: low: ≥ 0% medium: ≥ 75.0% high: ≥ 90.0%
Coverage Exec / Excl / Total
Lines: 100.0% 30 / 13 / 43
Functions: 100.0% 7 / 0 / 7
Branches: -% 0 / 26 / 26
kai/ukernels/matmul/pack/kai_rhs_pack_kxn_x16p2vlx2b_x16_x16_sme.c
Line Branch Exec Source
1 //
2 // SPDX-FileCopyrightText: Copyright 2024-2025 Arm Limited and/or its affiliates <open-source-office@arm.com>
3 //
4 // SPDX-License-Identifier: Apache-2.0
5 //
6
7 #if (!defined(__aarch64__) || !defined(__ARM_FEATURE_SVE2)) && !defined(_M_ARM64)
8 #error This file must be compiled for AArch64, FEAT_SVE2.
9 #else // Architectural features check.
10
11 #include "kai_rhs_pack_kxn_x16p2vlx2b_x16_x16_sme.h"
12
13 #include <stddef.h>
14 #include <stdint.h>
15 #include <string.h>
16
17 #include "kai/kai_common.h"
18
19 enum {
20 NR = 2,
21 KR = 2,
22 MAX_N_STEP = NR * ((KAI_SME_VEC_LENGTH_MAX_BYTES / sizeof(uint16_t)) / KR),
23 };
24
25 typedef struct {
26 const void* bias_ptr;
27 size_t width;
28 size_t height;
29 size_t in_stride;
30 size_t out_stride;
31 const void* in;
32 void* out;
33 const void* pad_row;
34 } KernelArgs;
35
36 static const size_t kai_num_bytes_input = sizeof(uint16_t);
37 static const size_t kai_num_bytes_output = sizeof(uint16_t);
38 static const size_t kai_num_bytes_bias = sizeof(uint16_t);
39
40 void kai_kernel_rhs_pack_kxn_x16p2vlx2b_x16_x16_sme(const KernelArgs* args_ptr);
41
42 5040 size_t kai_get_n_step_rhs_pack_kxn_x16p2vlx2b_x16_x16_sme(void) {
43 5040 return NR * kai_get_sme_vector_length_u16() / KR;
44 }
45
46 420 size_t kai_get_rhs_offset_rhs_pack_kxn_x16p2vlx2b_x16_x16_sme(size_t n_idx) {
47 − KAI_ASSUME(n_idx % kai_get_n_step_rhs_pack_kxn_x16p2vlx2b_x16_x16_sme() == 0);
48
49 420 return n_idx * kai_num_bytes_input;
50 }
51
52 528 size_t kai_get_bias_offset_rhs_pack_kxn_x16p2vlx2b_x16_x16_sme(size_t n_idx) {
53 528 return n_idx * kai_num_bytes_bias;
54 }
55
56 1260 size_t kai_get_rhs_packed_stride_rhs_pack_kxn_x16p2vlx2b_x16_x16_sme(size_t k) {
57 2520 return kai_get_n_step_rhs_pack_kxn_x16p2vlx2b_x16_x16_sme() *
58 1260 (kai_num_bytes_bias + kai_roundup(k, KR) * kai_num_bytes_output);
59 }
60
61 840 size_t kai_get_rhs_packed_offset_rhs_pack_kxn_x16p2vlx2b_x16_x16_sme(size_t n_idx, size_t k) {
62 − KAI_ASSUME(n_idx % kai_get_n_step_rhs_pack_kxn_x16p2vlx2b_x16_x16_sme() == 0);
63
64 840 const size_t block_idx = n_idx / kai_get_n_step_rhs_pack_kxn_x16p2vlx2b_x16_x16_sme();
65 1680 return block_idx * kai_get_rhs_packed_stride_rhs_pack_kxn_x16p2vlx2b_x16_x16_sme(k);
66 840 }
67
68 420 size_t kai_get_rhs_packed_size_rhs_pack_kxn_x16p2vlx2b_x16_x16_sme(size_t n, size_t k) {
69 420 const size_t n_nr_blocks = kai_roundup(n, kai_get_n_step_rhs_pack_kxn_x16p2vlx2b_x16_x16_sme());
70 840 return kai_get_rhs_packed_offset_rhs_pack_kxn_x16p2vlx2b_x16_x16_sme(n_nr_blocks, k);
71 420 }
72
73 420 void kai_run_rhs_pack_kxn_x16p2vlx2b_x16_x16_sme(
74 size_t num_groups, size_t n, size_t k, size_t nr, size_t kr, size_t sr, size_t rhs_stride_row, const void* rhs,
75 const void* bias, const void* scale, void* rhs_packed, size_t extra_bytes, const void* params) {
76 − KAI_ASSUME(num_groups == 1);
77 − KAI_ASSUME(nr == kai_get_n_step_rhs_pack_kxn_x16p2vlx2b_x16_x16_sme());
78 − KAI_ASSUME(kr == KR);
79 − KAI_ASSUME(sr == 1);
80 − KAI_ASSUME(rhs != NULL);
81 − KAI_ASSUME(bias != NULL);
82 − KAI_ASSUME(scale == NULL);
83 − KAI_ASSUME(rhs_packed != NULL);
84 − KAI_ASSUME(extra_bytes == 0);
85 − KAI_ASSUME(params == NULL);
86
87 − KAI_ASSERT(kai_get_n_step_rhs_pack_kxn_x16p2vlx2b_x16_x16_sme() <= MAX_N_STEP);
88 static const uint16_t pad_row[MAX_N_STEP] = {0};
89
90 420 KernelArgs args;
91 420 args.bias_ptr = bias;
92 420 args.height = k;
93 420 args.width = n;
94 420 args.in = rhs;
95 420 args.out = rhs_packed;
96 420 args.in_stride = rhs_stride_row;
97 420 args.out_stride = kai_get_rhs_packed_stride_rhs_pack_kxn_x16p2vlx2b_x16_x16_sme(args.height);
98 420 args.pad_row = pad_row;
99
100 420 kai_commit_za();
101
102 420 kai_kernel_rhs_pack_kxn_x16p2vlx2b_x16_x16_sme(&args);
103 420 }
104
105 #endif // Architectural features check.
106