Commit df7875e3 authored by Your Name's avatar Your Name

Fix litellm api_base for non-Ollama models

- Add logic to set api_base to server's own URL for non-Ollama models
- Extract host/port from request headers (X-Forwarded-For, Host header)
- Determine protocol (http/https) based on global_args
- Include debug output showing the determined api_base
- This ensures litellm can properly connect to local server when using litellm backend with local models
parent 6b297f18
This diff is collapsed.
...@@ -5230,6 +5230,35 @@ async def chat_completions(request: ChatCompletionRequest, http_request: Request ...@@ -5230,6 +5230,35 @@ async def chat_completions(request: ChatCompletionRequest, http_request: Request
port = getattr(global_args, 'port', 11434) if global_args else 11434 port = getattr(global_args, 'port', 11434) if global_args else 11434
api_base = f"http://{client_host}:{port}/v1" api_base = f"http://{client_host}:{port}/v1"
print(f"DEBUG: Using api_base for Ollama: {api_base}") print(f"DEBUG: Using api_base for Ollama: {api_base}")
else:
# For non-Ollama models, use the server's own URL as base
# This allows LiteLLM to make requests to the local server
if http_request:
# Get the host from the request headers
host_header = http_request.headers.get('host', '')
if host_header:
# Strip port if present to reconstruct clean URL
if ':' in host_header:
client_host = host_header.split(':')[0]
# Keep the port from the request for consistency
server_port = host_header.split(':')[1] if len(host_header.split(':')) > 1 else str(getattr(global_args, 'port', 6745))
else:
client_host = host_header
server_port = str(getattr(global_args, 'port', 6745))
else:
# Fallback to client host if no Host header
client_host = http_request.client.host if http_request.client else "127.0.0.1"
server_port = str(getattr(global_args, 'port', 6745))
else:
# Fallback if no http_request
client_host = "127.0.0.1"
server_port = str(getattr(global_args, 'port', 6745))
# Determine protocol (http or https)
use_https = getattr(global_args, 'https', False) or getattr(global_args, 'pubkey', None)
protocol = "https" if use_https else "http"
api_base = f"{protocol}://{client_host}:{server_port}/v1"
print(f"DEBUG: Using api_base for local server: {api_base}")
# Get or create litellm backend # Get or create litellm backend
litellm_backend = get_litellm_backend( litellm_backend = get_litellm_backend(
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment