From 23ce5b30afc7e3835cd5654cd289c70216a2ea29 Mon Sep 17 00:00:00 2001 From: Sean Campbell Date: Wed, 22 Apr 2026 21:15:57 -0600 Subject: [PATCH] cookbook: add custom fine-tuned GGUF routing via Ollama --- ...LiteLLM_Custom_FineTuned_GGUF_Ollama.ipynb | 192 ++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 cookbook/LiteLLM_Custom_FineTuned_GGUF_Ollama.ipynb diff --git a/cookbook/LiteLLM_Custom_FineTuned_GGUF_Ollama.ipynb b/cookbook/LiteLLM_Custom_FineTuned_GGUF_Ollama.ipynb new file mode 100644 index 00000000000..b04c5939bd3 --- /dev/null +++ b/cookbook/LiteLLM_Custom_FineTuned_GGUF_Ollama.ipynb @@ -0,0 +1,192 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Routing to a Custom Fine-Tuned GGUF Model via Ollama\n", + "\n", + "This cookbook shows how to use LiteLLM to route requests to a locally-deployed, custom fine-tuned GGUF model alongside cloud providers — treating your own model as a first-class citizen in a multi-provider setup.\n", + "\n", + "**Use case**: You have fine-tuned a small model (e.g., Qwen2.5 or Llama-3 base) on domain-specific data, exported it to GGUF, pushed it to Hugging Face Hub, and deployed it locally with Ollama. You want LiteLLM to route certain request types to your fine-tuned model and fall back to a cloud provider for others.\n", + "\n", + "## Prerequisites\n", + "\n", + "```bash\n", + "pip install litellm ollama\n", + "```\n", + "\n", + "Deploy your GGUF model with Ollama:\n", + "```bash\n", + "# Pull directly from Hugging Face Hub (no Modelfile needed for basic use)\n", + "ollama pull hf.co/your-username/your-model:Q4_K_M\n", + "\n", + "# Create a local alias\n", + "ollama cp hf.co/your-username/your-model:Q4_K_M my-model\n", + "\n", + "# Verify it's running\n", + "ollama run my-model \"Hello\"\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import litellm\n", + "from litellm import completion\n", + "\n", + "# LiteLLM routes to Ollama models using the 'ollama/' prefix.\n", + "# Replace 'my-model' with the name you gave your model in 'ollama cp'.\n", + "\n", + "response = completion(\n", + " model=\"ollama/my-model\",\n", + " messages=[{\"role\": \"user\", \"content\": \"What is the capital of France?\"}],\n", + " api_base=\"http://localhost:11434\", # default Ollama address\n", + ")\n", + "print(response.choices[0].message.content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Streaming responses from your fine-tuned model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = completion(\n", + " model=\"ollama/my-model\",\n", + " messages=[{\"role\": \"user\", \"content\": \"Explain quantum entanglement in one paragraph.\"}],\n", + " api_base=\"http://localhost:11434\",\n", + " stream=True,\n", + ")\n", + "for chunk in response:\n", + " print(chunk.choices[0].delta.content or \"\", end=\"\", flush=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Multi-provider routing: local fine-tuned model + cloud fallback\n", + "\n", + "Route domain-specific queries to your fine-tuned model and general queries to a cloud provider." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from litellm import Router\n", + "\n", + "router = Router(\n", + " model_list=[\n", + " {\n", + " \"model_name\": \"domain-expert\", # logical name your code uses\n", + " \"litellm_params\": {\n", + " \"model\": \"ollama/my-model\", # your fine-tuned GGUF via Ollama\n", + " \"api_base\": \"http://localhost:11434\",\n", + " },\n", + " },\n", + " {\n", + " \"model_name\": \"general\",\n", + " \"litellm_params\": {\n", + " \"model\": \"claude-haiku-4-5\", # cloud fallback\n", + " \"api_key\": os.environ.get(\"ANTHROPIC_API_KEY\"),\n", + " },\n", + " },\n", + " ],\n", + " routing_strategy=\"simple-shuffle\",\n", + ")\n", + "\n", + "# Route to your fine-tuned model\n", + "domain_response = router.completion(\n", + " model=\"domain-expert\",\n", + " messages=[{\"role\": \"user\", \"content\": \"Query specific to your fine-tuning domain\"}],\n", + ")\n", + "print(\"Fine-tuned:\", domain_response.choices[0].message.content)\n", + "\n", + "# Route to cloud for general tasks\n", + "general_response = router.completion(\n", + " model=\"general\",\n", + " messages=[{\"role\": \"user\", \"content\": \"Write a haiku about mountains.\"}],\n", + ")\n", + "print(\"Cloud:\", general_response.choices[0].message.content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Cost tracking for your local model\n", + "\n", + "LiteLLM can track usage for local models too — useful when comparing cost efficiency of your fine-tuned model vs cloud." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "litellm.success_callback = [\"langfuse\"] # or any other logger\n", + "\n", + "# Set a custom cost for your local model ($/1M tokens)\n", + "litellm.register_model({\n", + " \"ollama/my-model\": {\n", + " \"input_cost_per_token\": 0.0, # local inference = $0 per token\n", + " \"output_cost_per_token\": 0.0,\n", + " \"max_tokens\": 8192,\n", + " }\n", + "})\n", + "\n", + "response = completion(\n", + " model=\"ollama/my-model\",\n", + " messages=[{\"role\": \"user\", \"content\": \"Test\"}],\n", + " api_base=\"http://localhost:11434\",\n", + ")\n", + "print(f\"Tokens used: {response.usage.total_tokens}, Cost: ${response._hidden_params.get('response_cost', 0):.6f}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Tips for fine-tuned GGUF models\n", + "\n", + "- **Q4_K_M** is the sweet spot for most fine-tuned models: good quality, fast inference, ~40% of full-precision size.\n", + "- If your fine-tune used a custom chat template, set it in the Modelfile before running `ollama create`:\n", + " ```dockerfile\n", + " FROM hf.co/your-username/your-model:Q4_K_M\n", + " TEMPLATE \"{{ .System }}\\n{{ .Prompt }}\"\n", + " ```\n", + "- Use `ollama list` to confirm the model is registered before routing LiteLLM to it.\n", + "- If your model was trained with a specific system prompt, set it via `litellm_params.system_prompt` in the router config." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11.0" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +}