From 588318660c5076795ad43487c3410f1a44970bf8 Mon Sep 17 00:00:00 2001
From: sudoingX <200180104+sudoingX@users.noreply.github.com>
Date: Sat, 19 Sep 2026 08:22:44 +0000
Subject: [PATCH 5/5] Fix: use the mat-vec kernel for bf16 matrices under 64
 rows at 2 to 8 columns

The qwen35 gated-delta-net gate projections are bf16 [5120 x 48]. At 2 to 8
columns on Ampere they took a 10.5 us path where mul_mat_vec_f takes 4.4 to
7.3 us (test-backend-ops perf, RTX 3060); at one column both already used
mul_mat_vec_f (3.3 us). 96 calls per token on the 27B model.
---
 ggml/src/ggml-cuda/mmvf.cu | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/ggml/src/ggml-cuda/mmvf.cu b/ggml/src/ggml-cuda/mmvf.cu
index 9810f3edc..b12245e71 100644
--- a/ggml/src/ggml-cuda/mmvf.cu
+++ b/ggml/src/ggml-cuda/mmvf.cu
@@ -850,7 +850,9 @@ bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, const int64_t * src0
             if (GGML_CUDA_CC_IS_NVIDIA(cc)) {
                 const bool src0_small = (src0_ne[1] <= 512 || src0_ne[2]*src0_ne[3] == 1);
                 if (ampere_mma_available(cc)) {
-                    if (ggml_cuda_batch_invariant()) {
+                    // a few dozen rows (the qwen35 gated-delta-net gate projections) run faster as a
+                    // mat-vec than through the tensor-core path at 2 to 8 columns: 3.4 vs 10.5 us on an RTX 3060
+                    if (ggml_cuda_batch_invariant() || src0_ne[1] <= 64) {
                         return src0_small && ne11 <= MMVF_MAX_BATCH_SIZE;
                     }
                     return src0_small && ne11 == 1;
-- 
2.34.1