From 254725c6a23cd1639abc48503305052dcfe08894 Mon Sep 17 00:00:00 2001
From: sudoingX <200180104+sudoingX@users.noreply.github.com>
Date: Fri, 18 Sep 2026 21:36:39 +0000
Subject: [PATCH] Fix: apply the Hadamard inverse to token embeddings in the
 qwen35 MTP graph

---
 src/models/qwen35.cpp | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp
index 8e0944bc5..51e9298b1 100644
--- a/src/models/qwen35.cpp
+++ b/src/models/qwen35.cpp
@@ -1,5 +1,6 @@
 #include "models.h"
 #include "llama-memory-recurrent.h"
+#include "llama-impl.h"
 
 void llama_model_qwen35::load_arch_hparams(llama_model_loader & ml) {
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,       hparams.f_norm_rms_eps);
@@ -595,6 +596,20 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr
         ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd;
 
         tok_embd = ggml_get_rows(ctx0, tok_embd_w, inp->tokens);
+
+        // a Hadamard-latent embedding table (prism.hadamard.inverse_weight_names)
+        // stores rotated rows; restore the primal basis right after the lookup,
+        // the same way llm_graph_context::build_inp_embd() does, so an MTP head
+        // that shares the trunk's token_embd reads standard-basis embeddings
+        if (hadamard_inverses) {
+            const auto it = hadamard_inverses->find(tok_embd_w);
+            if (it != hadamard_inverses->end()) {
+                tok_embd = llama_mul_mat_hadamard(ctx0, tok_embd, it->second.rot);
+                if (it->second.signs) {
+                    tok_embd = ggml_mul(ctx0, tok_embd, it->second.signs);
+                }
+            }
+        }
     } else {
         tok_embd = inp->embd;
     }
-- 
2.34.1