slower but more stable now with fp32 accumulation

Signed-off-by: Karen Mosoyan <karen.mossoyan@gmail.com>

Author: kar-m

cactus/engine/engine_tokenizer.cpp

@@ -157,14 +157,7 @@ std::string Tokenizer::format_qwen_style(const std::vector<ChatMessage>& message
     }
 
     if (add_generation_prompt) {
-        const bool template_has_think = !chat_template_.empty() && chat_template_.find("<think>") != std::string::npos;
-        if (template_has_think) {
-            result += "<|im_start|>assistant\n<think>\n";
-        } else if (!tools_json.empty()) {
-            result += "<|im_start|>assistant\n<think>\n</think>\n\n";
-        } else {
-            result += "<|im_start|>assistant\n";
-        }
+        result += "<|im_start|>assistant\n<think>\n\n</think>\n\n";
     }
 
     return result;

cactus/graph/graph.h

@@ -549,10 +549,9 @@ class CactusGraph {
 
     size_t lstm_cell(size_t input, size_t h_prev, size_t c_prev, size_t weight_ih, size_t weight_hh, size_t bias_ih, size_t bias_hh);
     size_t gated_deltanet_decode(size_t query, size_t key, size_t value, size_t gate_log, size_t beta,
-                                 size_t initial_state, float scale = 0.0f, size_t num_qk_heads = 0);
+                                 size_t initial_state, float scale = 0.0f);
     size_t gated_deltanet_prefill(size_t query, size_t key, size_t value, size_t gate_log, size_t beta,
-                                  size_t initial_state, size_t chunk_size = 64, float scale = 0.0f,
-                                  size_t num_qk_heads = 0);
+                                  size_t initial_state, size_t chunk_size = 64, float scale = 0.0f);
     size_t stft(size_t input, size_t weight, size_t stride, size_t num_fft_bins);
 
     size_t sample(size_t logits, float temperature = 0.6f, float top_p = 0.95f, size_t top_k = 20,

cactus/graph/graph_builder.cpp

@@ -957,7 +957,7 @@ size_t CactusGraph::lstm_cell(size_t input, size_t h_prev, size_t c_prev, size_t
 }
 
 size_t CactusGraph::gated_deltanet_decode(size_t query, size_t key, size_t value, size_t gate_log, size_t beta,
-                                          size_t initial_state, float scale, size_t num_qk_heads) {
+                                          size_t initial_state, float scale) {
     const auto& q = get_output_buffer(query);
     const auto& k = get_output_buffer(key);
     const auto& v = get_output_buffer(value);
@@ -981,7 +981,9 @@ size_t CactusGraph::gated_deltanet_decode(size_t query, size_t key, size_t value
     const size_t K = q.shape[3];
     const size_t Hv = v.shape[2];
     const size_t V = v.shape[3];
-
+    if (T != 1) {
+        throw std::runtime_error("gated_deltanet_decode expects sequence length T=1");
+    }
     auto is_supported_precision = [](Precision p) {
         return p == Precision::FP16 || p == Precision::FP32;
     };
@@ -1007,8 +1009,7 @@ size_t CactusGraph::gated_deltanet_decode(size_t query, size_t key, size_t value
 }
 
 size_t CactusGraph::gated_deltanet_prefill(size_t query, size_t key, size_t value, size_t gate_log, size_t beta,
-                                           size_t initial_state, size_t chunk_size, float scale,
-                                           size_t num_qk_heads) {
+                                           size_t initial_state, size_t chunk_size, float scale) {
     const auto& q = get_output_buffer(query);
     const auto& k = get_output_buffer(key);
     const auto& v = get_output_buffer(value);
@@ -1032,7 +1033,6 @@ size_t CactusGraph::gated_deltanet_prefill(size_t query, size_t key, size_t valu
     const size_t K = q.shape[3];
     const size_t Hv = v.shape[2];
     const size_t V = v.shape[3];
-
     auto is_supported_precision = [](Precision p) {
         return p == Precision::FP16 || p == Precision::FP32;
     };

cactus/graph/graph_ops_nn.cpp

@@ -2078,7 +2078,7 @@ void compute_groupnorm_node(GraphNode& node, const std::vector<std::unique_ptr<G
             }
         }
     }
-
+}
 
 void compute_lstm_cell_node(GraphNode& node, const std::vector<std::unique_ptr<GraphNode>>& nodes, const std::unordered_map<size_t, size_t>& node_index_map) {
     const auto& input_buffer = nodes[node_index_map.at(node.input_ids[0])]->output_buffer;
@@ -2137,4 +2137,3 @@ void compute_lstm_cell_node(GraphNode& node, const std::vector<std::unique_ptr<G
         }
     }
 }
-}

cactus/models/model_qwen3p5.cpp

@@ -160,7 +160,7 @@ void Qwen3p5Model::post_init() {
         throw std::runtime_error("Qwen3p5Model computed zero cache row dim");
     }
 
-    conv_cache_.init(config_.num_layers, deltanet_cache_row_dim_, 1, Precision::FP16);
+    conv_cache_.init(config_.num_layers, deltanet_cache_row_dim_, 1, Precision::FP32);
     last_forward_used_cache_ = false;
     deltanet_total_seq_len_ = 0;
 }
@@ -451,6 +451,7 @@ size_t Qwen3p5Model::build_gated_deltanet(CactusGraph* gb, size_t normalized_inp
             size_t history_2d = 0;
             if (prev_conv_flat != 0) {
                 history_2d = gb->reshape(prev_conv_flat, {deltanet_conv_history_len_, mixed_proj_dim});
+                history_2d = gb->precision_cast(history_2d, Precision::FP16);
             } else {
                 history_2d = gb->input({deltanet_conv_history_len_, mixed_proj_dim}, Precision::FP16);
                 std::vector<__fp16> zeros(deltanet_conv_flat_dim_, static_cast<__fp16>(0.0f));
@@ -532,11 +533,11 @@ size_t Qwen3p5Model::build_gated_deltanet(CactusGraph* gb, size_t normalized_inp
 
     size_t deltanet_out;
     if (use_cache && seq_len == 1) {
-        deltanet_out = gb->gated_deltanet_decode(q_4d, k_4d, v_4d, gate_3d, beta_3d, initial_state, 0.0f, num_k_heads);
+        deltanet_out = gb->gated_deltanet_decode(q_4d, k_4d, v_4d, gate_3d, beta_3d, initial_state, 0.0f);
     } else {
         const size_t chunk_for_op = std::min<size_t>(64, std::max<size_t>(1, seq_len));
         deltanet_out = gb->gated_deltanet_prefill(
-            q_4d, k_4d, v_4d, gate_3d, beta_3d, initial_state, chunk_for_op, 0.0f, num_k_heads);
+            q_4d, k_4d, v_4d, gate_3d, beta_3d, initial_state, chunk_for_op, 0.0f);
     }
 
     size_t y_4d = gb->slice(deltanet_out, 1, 0, seq_len);
@@ -560,6 +561,7 @@ size_t Qwen3p5Model::build_gated_deltanet(CactusGraph* gb, size_t normalized_inp
                 }
             } else if (prev_conv_flat != 0) {
                 history_2d = gb->reshape(prev_conv_flat, {deltanet_conv_history_len_, mixed_proj_dim});
+                history_2d = gb->precision_cast(history_2d, Precision::FP16);
             } else {
                 history_2d = gb->input({deltanet_conv_history_len_, mixed_proj_dim}, Precision::FP16);
                 std::vector<__fp16> zeros(deltanet_conv_flat_dim_, static_cast<__fp16>(0.0f));
@@ -569,6 +571,7 @@ size_t Qwen3p5Model::build_gated_deltanet(CactusGraph* gb, size_t normalized_inp
             size_t history_flat = gb->reshape(history_2d, {1, deltanet_conv_flat_dim_});
             packed_cache = gb->concat(packed_cache, history_flat, 1);
         }
+        packed_cache = gb->precision_cast(packed_cache, conv_cache_.precision);
         conv_cache_state_nodes_[layer_idx] = packed_cache;
         cache_k_output_nodes_[layer_idx] = 0;
         cache_v_output_nodes_[layer_idx] = 0;