Commit 280145ee authored by Your Name's avatar Your Name

Fix: Use GGUF embedded chat template via llama.cpp instead of manual formatting

When chat_template is 'default', it means llama.cpp detected an embedded
template in the GGUF model. Don't fall back to manual formatting - instead
let llama.cpp's create_chat_completion use its internal template handling.
parent 90abc6a8
......@@ -2019,8 +2019,9 @@ class VulkanBackend(ModelBackend):
# Check if we should use manual formatting based on detected template
# Always use manual formatting when tools are present, since Jinja templates often fail with tool messages
# Also use manual formatting when we have a known template but no HuggingFace tokenizer
# NOTE: "default" means llama.cpp detected an embedded template - let llama.cpp handle it via create_chat_completion
use_manual = self.chat_template in ("unknown", "jinja_fallback", None) or tools is not None
use_manual = use_manual or (self.chat_template is not None and self.hf_tokenizer is None)
use_manual = use_manual or (self.chat_template is not None and self.chat_template != "default" and self.hf_tokenizer is None)
use_hf = self.hf_tokenizer is not None
if use_hf:
......@@ -2092,8 +2093,9 @@ class VulkanBackend(ModelBackend):
# Check if we should use manual formatting based on detected template
# Always use manual formatting when tools are present, since Jinja templates often fail with tool messages
# Also use manual formatting when we have a known template but no HuggingFace tokenizer
# NOTE: "default" means llama.cpp detected an embedded template - let llama.cpp handle it via create_chat_completion
use_manual = self.chat_template in ("unknown", "jinja_fallback", None) or tools is not None
use_manual = use_manual or (self.chat_template is not None and self.hf_tokenizer is None)
use_manual = use_manual or (self.chat_template is not None and self.chat_template != "default" and self.hf_tokenizer is None)
use_hf = self.hf_tokenizer is not None
if use_hf:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment