Commit 47abbabb authored by Your Name's avatar Your Name

Fix raw mode variable initialization

Fixed issue where raw mode variables were being re-initialized,
which was overwriting the values set in the prompt handling section.
parent ceb4ae88
...@@ -2170,52 +2170,50 @@ async def chat_completions(request: ChatCompletionRequest, http_request: Request ...@@ -2170,52 +2170,50 @@ async def chat_completions(request: ChatCompletionRequest, http_request: Request
}) })
# Handle raw mode - use generate() instead of generate_chat() for raw prompt completion # Handle raw mode - use generate() instead of generate_chat() for raw prompt completion
use_raw_mode = False # Note: These may have been set earlier in the prompt handling section
raw_prompt_for_generation = None # Initialize only if not already set
raw_stop_sequences = None if 'use_raw_mode' not in locals():
use_raw_mode = False
if 'raw_prompt_for_generation' not in locals():
raw_prompt_for_generation = None
if 'raw_stop_sequences' not in locals():
raw_stop_sequences = None
# Check if we set raw mode in the prompt handling section above # Check if we need to set up raw mode (if not already done in prompt handling)
# The variables should already be set if raw was in force_reasoning_args if "raw" in force_reasoning_args and not use_raw_mode:
if "raw" in force_reasoning_args: # Set up raw mode using tokenizer
# Raw mode was already set up in the prompt handling section tokenizer = None
# Just verify the variables exist if hasattr(current_manager, 'backend') and hasattr(current_manager.backend, 'tokenizer'):
try: tokenizer = current_manager.backend.tokenizer
_ = raw_prompt_for_generation
_ = raw_stop_sequences if tokenizer is not None:
except NameError: # Extract system and user messages
# Variables not set - try to get tokenizer again system_prompt = "You are a helpful assistant."
tokenizer = None user_message = ""
if hasattr(current_manager, 'backend') and hasattr(current_manager.backend, 'tokenizer'): for msg in messages:
tokenizer = current_manager.backend.tokenizer if msg.role == "system":
system_prompt = msg.content
elif msg.role == "user":
user_message = msg.content
if tokenizer is not None: # Get the prompt with generation prompt
# Extract system and user messages try:
system_prompt = "You are a helpful assistant." raw_prompt_for_generation = tokenizer.apply_chat_template(
user_message = "" [{"role": "system", "content": system_prompt},
for msg in messages: {"role": "user", "content": user_message}],
if msg.role == "system": add_generation_prompt=True,
system_prompt = msg.content tokenize=False
elif msg.role == "user": )
user_message = msg.content except Exception as e:
raw_prompt_for_generation = f"System: {system_prompt}\n\nUser: {user_message}\n\nAssistant:"
# Get the prompt with generation prompt
try: # Get reasoning tag
raw_prompt_for_generation = tokenizer.apply_chat_template( thought_tag, close_tag, _ = get_reasoning_stop_tokens(model_family)
[{"role": "system", "content": system_prompt}, raw_prompt_for_generation += thought_tag + "Let me think about this step by step."
{"role": "user", "content": user_message}], raw_stop_sequences = list(stop_sequences)
add_generation_prompt=True, if close_tag not in raw_stop_sequences:
tokenize=False raw_stop_sequences.append(close_tag)
) use_raw_mode = True
except Exception as e:
raw_prompt_for_generation = f"System: {system_prompt}\n\nUser: {user_message}\n\nAssistant:"
# Get reasoning tag
thought_tag, close_tag, _ = get_reasoning_stop_tokens(model_family)
raw_prompt_for_generation += thought_tag + "Let me think about this step by step."
raw_stop_sequences = list(stop_sequences)
if close_tag not in raw_stop_sequences:
raw_stop_sequences.append(close_tag)
use_raw_mode = True
# Get resolved model name for response (with coderai/ prefix and proper formatting) # Get resolved model name for response (with coderai/ prefix and proper formatting)
response_model_name = get_resolved_model_name(requested_model, current_manager) response_model_name = get_resolved_model_name(requested_model, current_manager)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment