Commit 13e81d0d authored by Your Name's avatar Your Name

Fix HF tokenizer loading to check for cached local file first when model is URL

parent b4d3d43b
......@@ -1384,6 +1384,13 @@ class VulkanBackend(ModelBackend):
print("DEBUG: No model name available for HuggingFace tokenizer")
return
# If model_path is a URL, try to get the cached local path first
if model_path.startswith('http://') or model_path.startswith('https://'):
cached_path = get_cached_model_path(model_path)
if cached_path and os.path.exists(cached_path):
model_path = cached_path
print(f"DEBUG: Using cached model path for HF tokenizer: {model_path}")
try:
from transformers import AutoTokenizer
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment