Skip to content

C
coderai
Commit 10dc9f5c authored by Stefy Lanza (nextime / spora )'s avatar Stefy Lanza (nextime / spora )
Browse files
Options

Make --model optional when --audio-model or --image-model are specified 

- --model is now optional if using audio or image models only
- Shows helpful error message with examples if no model specified
- Prints available models at startup
parent 3ae1869a
Hide whitespace changes
Inline Side-by-side
Showing with 64 additions and 55 deletions
coderai
View file @ 10dc9f5c
...@@ -2999,7 +2999,7 @@ def parse_args(): ...@@ -2999,7 +2999,7 @@ def parse_args():
"--model", "--model",
type=str, type=str,
default=None, default=None,
help="Model name or path. For NVIDIA: HuggingFace model. For Vulkan: GGUF file path or HF repo", help="Model name, path, or URL for text-to-text LLM. Optional if only using --audio-model or --image-model",
) )
parser.add_argument( parser.add_argument(
"--backend", "--backend",
...@@ -3186,8 +3186,10 @@ def main(): ...@@ -3186,8 +3186,10 @@ def main():
# Get model name from args or prompt interactively # Get model name from args or prompt interactively
model_name = args.model model_name = args.model
if model_name is None:
print("No model specified. Please enter a model name.") # Validate: must have at least one model specified
if model_name is None and args.audio_model is None and args.image_model is None:
print("Error: At least one of --model, --audio-model, or --image-model must be specified.")
print("") print("")
print("For NVIDIA backend (HuggingFace models):") print("For NVIDIA backend (HuggingFace models):")
print(" - microsoft/DialoGPT-medium") print(" - microsoft/DialoGPT-medium")
...@@ -3197,12 +3199,14 @@ def main(): ...@@ -3197,12 +3199,14 @@ def main():
print("For Vulkan backend (GGUF models):") print("For Vulkan backend (GGUF models):")
print(" - Local path: ./phi-3-mini-4k-instruct-q4_k_m.gguf") print(" - Local path: ./phi-3-mini-4k-instruct-q4_k_m.gguf")
print(" - HuggingFace: microsoft/Phi-3-mini-4k-instruct-gguf") print(" - HuggingFace: microsoft/Phi-3-mini-4k-instruct-gguf")
print(" - URL: https://huggingface.co/.../model.gguf")
print("") print("")
model_name = input("Enter model name: ").strip() print("For audio transcription:")
print(" - --audio-model base")
if not model_name: print("")
print("Error: Model name is required") print("For image generation:")
sys.exit(1) print(" - --image-model stabilityai/stable-diffusion-xl-base-1.0")
sys.exit(1)
# Detect available backends # Detect available backends
available = detect_available_backends() available = detect_available_backends()
...@@ -3212,52 +3216,56 @@ def main(): ...@@ -3212,52 +3216,56 @@ def main():
print(f" [{status}] {name}") print(f" [{status}] {name}")
print("") print("")
# Load the main model # Load the main model (only if specified)
load_kwargs = { if model_name:
'offload_dir': args.offload_dir, load_kwargs = {
'load_in_4bit': args.load_in_4bit, 'offload_dir': args.offload_dir,
'load_in_8bit': args.load_in_8bit, 'load_in_4bit': args.load_in_4bit,
'manual_ram_gb': args.ram, 'load_in_8bit': args.load_in_8bit,
'flash_attn': args.flash_attn, 'manual_ram_gb': args.ram,
'offload_strategy': args.offload_strategy, 'flash_attn': args.flash_attn,
'max_gpu_percent': args.max_gpu_percent, 'offload_strategy': args.offload_strategy,
'n_gpu_layers': args.n_gpu_layers, 'max_gpu_percent': args.max_gpu_percent,
'n_ctx': args.n_ctx, 'n_gpu_layers': args.n_gpu_layers,
'main_gpu': args.vulkan_device, 'n_ctx': args.n_ctx,
'single_gpu': args.vulkan_single_gpu, 'main_gpu': args.vulkan_device,
} 'single_gpu': args.vulkan_single_gpu,
}
try:
model_manager.load_model( try:
model_name=model_name, model_manager.load_model(
backend_type=args.backend, model_name=model_name,
**load_kwargs backend_type=args.backend,
) **load_kwargs
# Register with multi_model_manager )
multi_model_manager.set_default_model(model_name, load_kwargs) # Register with multi_model_manager
multi_model_manager.add_model(model_name, model_manager) multi_model_manager.set_default_model(model_name, load_kwargs)
except Exception as e: multi_model_manager.add_model(model_name, model_manager)
print(f"\nError loading model: {e}") print(f"\nMain text model loaded: {model_name}")
error_str = str(e).lower() except Exception as e:
print("\nTroubleshooting:") print(f"\nError loading model: {e}")
if args.backend == "vulkan": error_str = str(e).lower()
print(" - For Vulkan, ensure you have Vulkan drivers installed") print("\nTroubleshooting:")
print(" - Make sure you're using a GGUF format model") if args.backend == "vulkan":
print(" - Run build.sh with 'vulkan' argument first") print(" - For Vulkan, ensure you have Vulkan drivers installed")
else: print(" - Make sure you're using a GGUF format model")
print(" - For NVIDIA, ensure PyTorch with CUDA is installed") print(" - Run build.sh with 'vulkan' argument first")
print(" - Run build.sh with 'nvidia' argument first") else:
if "tokenizer" in error_str or "sentencepiece" in error_str or "tiktoken" in error_str: print(" - For NVIDIA, ensure PyTorch with CUDA is installed")
print(" - Tokenizer error: ensure sentencepiece and tiktoken are installed") print(" - Run build.sh with 'nvidia' argument first")
print(" pip install sentencepiece tiktoken tokenizers") if "tokenizer" in error_str or "sentencepiece" in error_str or "tiktoken" in error_str:
# Check if trying to load GGUF model with NVIDIA backend print(" - Tokenizer error: ensure sentencepiece and tiktoken are installed")
if "gguf" in model_name.lower(): print(" pip install sentencepiece tiktoken tokenizers")
print(f"\n *** IMPORTANT: '{model_name}' appears to be a GGUF model ***") # Check if trying to load GGUF model with NVIDIA backend
print(" GGUF models are NOT compatible with the NVIDIA backend.") if "gguf" in model_name.lower():
print(" Use --backend vulkan instead, or choose a HuggingFace Transformers model.") print(f"\n *** IMPORTANT: '{model_name}' appears to be a GGUF model ***")
print("\n Example Vulkan command:") print(" GGUF models are NOT compatible with the NVIDIA backend.")
print(f" coderai --backend vulkan --model {model_name}") print(" Use --backend vulkan instead, or choose a HuggingFace Transformers model.")
sys.exit(1) print("\n Example Vulkan command:")
print(f" coderai --backend vulkan --model {model_name}")
sys.exit(1)
else:
print("\nNo main text model specified (--model). Running with audio/image models only.")
# Set up audio model if specified # Set up audio model if specified
if args.audio_model: if args.audio_model:
...@@ -3291,7 +3299,8 @@ def main(): ...@@ -3291,7 +3299,8 @@ def main():
import uvicorn import uvicorn
print(f"\nStarting server on http://{args.host}:{args.port}") print(f"\nStarting server on http://{args.host}:{args.port}")
print(f"API documentation available at http://{args.host}:{args.port}/docs") print(f"API documentation available at http://{args.host}:{args.port}/docs")
print(f"Using backend: {model_manager.backend_type}") if model_manager.backend is not None:
print(f"Using backend: {model_manager.backend_type}")
# Print available models # Print available models
models = multi_model_manager.list_models() models = multi_model_manager.list_models()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment