kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qai4c32p/kai_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod.c

Directory:	./
Coverage:	low: ≥ 0% medium: ≥ 75.0% high: ≥ 90.0%

	Coverage	Exec / Excl / Total
Lines:	98.2%	55 / 11 / 67
Functions:	100.0%	16 / 0 / 16
Branches:	50.0%	1 / 22 / 24

    kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qai4c32p/kai_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod.c
    
        Line
        Branch
        Exec
        Source
      
        //
      
        // SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its affiliates <open-source-office@arm.com>
      
        //
      
        // SPDX-License-Identifier: Apache-2.0
      
        //
      
        #if !defined(__aarch64__) && !defined(__ARM_FEATURE_DOTPROD) && !defined(_M_ARM64)
      
        #error "Dotprod extension required to compile this micro-kernel"
      
        #else  // Architectural features check.
      
        #include "kai_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod.h"
      
        #include <stddef.h>
      
        #include "kai/kai_common.h"
      
        typedef struct {
      
            float* dst;
      
            const void* lhs_packed;
      
            const void* rhs_packed;
      
            const float* clamp_vals;
      
            size_t dst_stride_row;
      
            size_t m;
      
            size_t n;
      
            size_t num_blocks;
      
            size_t num_subblocks;
      
        } KernelArgs;
      
        void kai_kernel_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod(KernelArgs* args_ptr);
      
        // Compute args
      
        static const size_t kai_m_step = 8;
      
        static const size_t kai_n_step = 4;
      
        // Packing args
      
        static const size_t kai_mr = 4;
      
        static const size_t kai_nr = 4;
      
        static const size_t kai_kr = 8;
      
        static const size_t kai_sr = 2;
      
        // LHS format args (num. bytes per value, multiplier, zero_point (if asymmetric))
      
        static const size_t kai_num_bytes_qvalue_lhs = 1;
      
        static const size_t kai_num_bytes_multiplier_lhs = 4;
      
        static const size_t kai_num_bytes_sum_lhs = 4;
      
        // RHS format args (num. bytes per value, multiplier, zero_point (if asymmetric), and reduction sum (if LHS is
      
        // asymmetric))
      
        static const size_t kai_num_bytes_recip_qvalue_rhs = 2;
      
        static const size_t kai_num_bytes_multiplier_rhs = 4;
      
        static const size_t kai_num_bytes_offset_rhs = 4;
      
        // DST format args
      
        static const size_t kai_num_bytes_dst_value = 4;
      
        // Extra args
      
        static const size_t kai_num_bytes_bias = 4;
      
        static const size_t kai_bl = 32;
      
        3108
        inline static size_t kai_get_num_bytes_per_block_lhs(size_t bl) {
      
        3108
            return (bl * kai_num_bytes_qvalue_lhs) + kai_num_bytes_multiplier_lhs + kai_num_bytes_sum_lhs;
      
        }
      
        3108
        inline static size_t kai_get_num_bytes_per_block_rhs(size_t bl) {
      
        −
            KAI_ASSUME((bl % kai_bl) == 0);
      
        6216
            size_t num_bytes_per_block_rhs =
      
        3108
                (bl / kai_num_bytes_recip_qvalue_rhs) + kai_num_bytes_multiplier_rhs + kai_num_bytes_offset_rhs;
      
        6216
            return num_bytes_per_block_rhs;
      
        3108
        }
      
        9330
        inline static size_t kai_get_num_blocks_per_row(size_t k, size_t bl) {
      
        −
            KAI_ASSUME((bl % kai_bl) == 0);
      
        9330
            return kai_roundup(k, bl) / bl;
      
        }
      
        3108
        inline static size_t kai_get_lhs_packed_stride(size_t k, size_t bl) {
      
        3108
            return kai_mr * kai_get_num_blocks_per_row(k, bl) * kai_get_num_bytes_per_block_lhs(bl);
      
        }
      
        3108
        inline static size_t kai_get_rhs_packed_stride(size_t k, size_t bl) {
      
        −
            KAI_ASSUME((bl % kai_bl) == 0);
      
        3108
            const size_t num_blocks_per_row = kai_get_num_blocks_per_row(k, bl);
      
        3108
            const size_t num_bytes_per_block = kai_get_num_bytes_per_block_rhs(bl);
      
        3108
            size_t rhs_packed_stride = kai_nr * (num_bytes_per_block * num_blocks_per_row);
      
            // Since the bias is packed with the RHS matrix, the stride is adjusted with the number of bytes of the bias
      
        3108
            rhs_packed_stride += kai_nr * kai_num_bytes_bias;
      
        6216
            return rhs_packed_stride;
      
        3108
        }
      
        6216
        size_t kai_get_m_step_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod(void) {
      
        6216
            return kai_m_step;
      
        }
      
        6216
        size_t kai_get_n_step_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod(void) {
      
        6216
            return kai_n_step;
      
        }
      
        6216
        size_t kai_get_mr_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod(void) {
      
        6216
            return kai_mr;
      
        }
      
        6216
        size_t kai_get_nr_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod(void) {
      
        6216
            return kai_nr;
      
        }
      
        6216
        size_t kai_get_kr_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod(void) {
      
        6216
            return kai_kr;
      
        }
      
        6216
        size_t kai_get_sr_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod(void) {
      
        6216
            return kai_sr;
      
        }
      
        3108
        size_t kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod(
      
            size_t m_idx, size_t k, size_t bl) {
      
        −
            KAI_ASSUME((m_idx % kai_m_step) == 0);
      
        3108
            return (m_idx / kai_mr) * kai_get_lhs_packed_stride(k, bl);
      
        }
      
        3108
        size_t kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod(
      
            size_t n_idx, size_t k, size_t bl) {
      
        −
            KAI_ASSUME((k % bl) == 0);
      
        −
            KAI_ASSUME((n_idx % kai_n_step) == 0);
      
        3108
            return (n_idx / kai_nr) * kai_get_rhs_packed_stride(k, bl);
      
        }
      
        3108
        size_t kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod(
      
            size_t m_idx, size_t n_idx, size_t dst_stride) {
      
        −
            KAI_ASSUME((m_idx % kai_m_step) == 0);
      
        −
            KAI_ASSUME((n_idx % kai_n_step) == 0);
      
        3108
            return (n_idx * kai_num_bytes_dst_value) + m_idx * dst_stride;
      
        }
      
        3108
        size_t kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod(size_t m, size_t n) {
      
        3108
            return m * n * kai_num_bytes_dst_value;
      
        }
      
        3114
        void kai_run_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod(
      
            size_t m,                         //
      
            size_t n,                         //
      
            size_t k,                         //
      
            size_t bl,                        //
      
            const void* restrict lhs_packed,  //
      
            const void* restrict rhs_packed,  //
      
            float* restrict dst,              // NOLINT(readability-non-const-parameter)
      
            size_t dst_stride_row,            //
      
            size_t dst_stride_col,            //
      
            float scalar_min,                 //
      
            float scalar_max) {
      
        −
            KAI_ASSUME(dst_stride_col == sizeof(float));
      
        −
            KAI_ASSUME((k % bl) == 0);
      
        −
            KAI_ASSUME((bl % kai_bl) == 0);
      
          1/2✓ Branch 0 taken 3114 times.
✗ Branch 1 not taken.

        3114
            if (m == 0) {
      
        ✗
                return;
      
            }
      
        3114
            const size_t num_subblocks = bl / kai_bl;
      
        3114
            const size_t num_blocks = kai_get_num_blocks_per_row(k, bl);
      
        3114
            const float clamp_vals[2] = {scalar_min, scalar_max};
      
        3114
            KernelArgs args;
      
        3114
            args.dst = dst;
      
        3114
            args.lhs_packed = lhs_packed;
      
        3114
            args.rhs_packed = rhs_packed;
      
        3114
            args.clamp_vals = clamp_vals;
      
        3114
            args.dst_stride_row = dst_stride_row;
      
        3114
            args.m = m;
      
        3114
            args.n = n;
      
        3114
            args.num_blocks = num_blocks;
      
        3114
            args.num_subblocks = num_subblocks;
      
        3114
            kai_kernel_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod(&args);
      
        3114
        }
      
        #endif  // Architectural features check.

Line	Branch	Exec	Source
1			//
2			// SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its affiliates <open-source-office@arm.com>
3			//
4			// SPDX-License-Identifier: Apache-2.0
5			//
6			#if !defined(__aarch64__) && !defined(__ARM_FEATURE_DOTPROD) && !defined(_M_ARM64)
7			#error "Dotprod extension required to compile this micro-kernel"
8			#else // Architectural features check.
9
10			#include "kai_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod.h"
11
12			#include <stddef.h>
13
14			#include "kai/kai_common.h"
15
16			typedef struct {
17			float* dst;
18			const void* lhs_packed;
19			const void* rhs_packed;
20			const float* clamp_vals;
21			size_t dst_stride_row;
22			size_t m;
23			size_t n;
24			size_t num_blocks;
25			size_t num_subblocks;
26			} KernelArgs;
27
28			void kai_kernel_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod(KernelArgs* args_ptr);
29
30			// Compute args
31			static const size_t kai_m_step = 8;
32			static const size_t kai_n_step = 4;
33			// Packing args
34			static const size_t kai_mr = 4;
35			static const size_t kai_nr = 4;
36			static const size_t kai_kr = 8;
37			static const size_t kai_sr = 2;
38			// LHS format args (num. bytes per value, multiplier, zero_point (if asymmetric))
39			static const size_t kai_num_bytes_qvalue_lhs = 1;
40			static const size_t kai_num_bytes_multiplier_lhs = 4;
41			static const size_t kai_num_bytes_sum_lhs = 4;
42			// RHS format args (num. bytes per value, multiplier, zero_point (if asymmetric), and reduction sum (if LHS is
43			// asymmetric))
44			static const size_t kai_num_bytes_recip_qvalue_rhs = 2;
45			static const size_t kai_num_bytes_multiplier_rhs = 4;
46			static const size_t kai_num_bytes_offset_rhs = 4;
47
48			// DST format args
49			static const size_t kai_num_bytes_dst_value = 4;
50			// Extra args
51			static const size_t kai_num_bytes_bias = 4;
52			static const size_t kai_bl = 32;
53
54		3108	inline static size_t kai_get_num_bytes_per_block_lhs(size_t bl) {
55		3108	return (bl * kai_num_bytes_qvalue_lhs) + kai_num_bytes_multiplier_lhs + kai_num_bytes_sum_lhs;
56			}
57
58		3108	inline static size_t kai_get_num_bytes_per_block_rhs(size_t bl) {
59		−	KAI_ASSUME((bl % kai_bl) == 0);
60		6216	size_t num_bytes_per_block_rhs =
61		3108	(bl / kai_num_bytes_recip_qvalue_rhs) + kai_num_bytes_multiplier_rhs + kai_num_bytes_offset_rhs;
62		6216	return num_bytes_per_block_rhs;
63		3108	}
64
65		9330	inline static size_t kai_get_num_blocks_per_row(size_t k, size_t bl) {
66		−	KAI_ASSUME((bl % kai_bl) == 0);
67
68		9330	return kai_roundup(k, bl) / bl;
69			}
70
71		3108	inline static size_t kai_get_lhs_packed_stride(size_t k, size_t bl) {
72		3108	return kai_mr * kai_get_num_blocks_per_row(k, bl) * kai_get_num_bytes_per_block_lhs(bl);
73			}
74
75		3108	inline static size_t kai_get_rhs_packed_stride(size_t k, size_t bl) {
76		−	KAI_ASSUME((bl % kai_bl) == 0);
77
78		3108	const size_t num_blocks_per_row = kai_get_num_blocks_per_row(k, bl);
79		3108	const size_t num_bytes_per_block = kai_get_num_bytes_per_block_rhs(bl);
80
81		3108	size_t rhs_packed_stride = kai_nr * (num_bytes_per_block * num_blocks_per_row);
82			// Since the bias is packed with the RHS matrix, the stride is adjusted with the number of bytes of the bias
83		3108	rhs_packed_stride += kai_nr * kai_num_bytes_bias;
84
85		6216	return rhs_packed_stride;
86		3108	}
87
88		6216	size_t kai_get_m_step_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod(void) {
89		6216	return kai_m_step;
90			}
91
92		6216	size_t kai_get_n_step_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod(void) {
93		6216	return kai_n_step;
94			}
95
96		6216	size_t kai_get_mr_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod(void) {
97		6216	return kai_mr;
98			}
99
100		6216	size_t kai_get_nr_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod(void) {
101		6216	return kai_nr;
102			}
103
104		6216	size_t kai_get_kr_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod(void) {
105		6216	return kai_kr;
106			}
107
108		6216	size_t kai_get_sr_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod(void) {
109		6216	return kai_sr;
110			}
111
112		3108	size_t kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod(
113			size_t m_idx, size_t k, size_t bl) {
114		−	KAI_ASSUME((m_idx % kai_m_step) == 0);
115
116		3108	return (m_idx / kai_mr) * kai_get_lhs_packed_stride(k, bl);
117			}
118
119		3108	size_t kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod(
120			size_t n_idx, size_t k, size_t bl) {
121		−	KAI_ASSUME((k % bl) == 0);
122		−	KAI_ASSUME((n_idx % kai_n_step) == 0);
123
124		3108	return (n_idx / kai_nr) * kai_get_rhs_packed_stride(k, bl);
125			}
126
127		3108	size_t kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod(
128			size_t m_idx, size_t n_idx, size_t dst_stride) {
129		−	KAI_ASSUME((m_idx % kai_m_step) == 0);
130		−	KAI_ASSUME((n_idx % kai_n_step) == 0);
131
132		3108	return (n_idx * kai_num_bytes_dst_value) + m_idx * dst_stride;
133			}
134
135		3108	size_t kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod(size_t m, size_t n) {
136		3108	return m * n * kai_num_bytes_dst_value;
137			}
138
139		3114	void kai_run_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod(
140			size_t m, //
141			size_t n, //
142			size_t k, //
143			size_t bl, //
144			const void* restrict lhs_packed, //
145			const void* restrict rhs_packed, //
146			float* restrict dst, // NOLINT(readability-non-const-parameter)
147			size_t dst_stride_row, //
148			size_t dst_stride_col, //
149			float scalar_min, //
150			float scalar_max) {
151		−	KAI_ASSUME(dst_stride_col == sizeof(float));
152		−	KAI_ASSUME((k % bl) == 0);
153		−	KAI_ASSUME((bl % kai_bl) == 0);
154	1/2 ✓ Branch 0 taken 3114 times. ✗ Branch 1 not taken.	3114	if (m == 0) {
155		✗	return;
156			}
157		3114	const size_t num_subblocks = bl / kai_bl;
158		3114	const size_t num_blocks = kai_get_num_blocks_per_row(k, bl);
159		3114	const float clamp_vals[2] = {scalar_min, scalar_max};
160
161		3114	KernelArgs args;
162
163		3114	args.dst = dst;
164		3114	args.lhs_packed = lhs_packed;
165		3114	args.rhs_packed = rhs_packed;
166		3114	args.clamp_vals = clamp_vals;
167		3114	args.dst_stride_row = dst_stride_row;
168		3114	args.m = m;
169		3114	args.n = n;
170		3114	args.num_blocks = num_blocks;
171		3114	args.num_subblocks = num_subblocks;
172
173		3114	kai_kernel_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod(&args);
174		3114	}
175
176			#endif // Architectural features check.
177

KleidiAI Coverage Report