scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-10 08:28:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 108

fix: resolve tealtiger ruff violations

Browse source
This commit is contained in:
AlphaRex-pixel 2026-08-14 21:39:30 +05:30
parent 9958d92d0b
commit b7ecab516a
92 changed files with 8940 additions and 4627 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/scripts/scan_keywords.py vendored
View file

@ -2,8 +2,8 @@
import json
import os
import sys
import urllib.request
import urllib.error
import urllib.request
def read_event_payload() -> dict:

4
.github/workflows/auto_update_price_and_context_window_file.py vendored
View file

@ -1,7 +1,9 @@
import asyncio
import aiohttp
import json
import aiohttp
# Asynchronously fetch data from a given URL
async def fetch_data(url):
try:

7
.github/workflows/run_llm_translation_tests.py vendored
View file

@ -6,15 +6,16 @@ This script runs the LLM translation tests and generates a comprehensive
markdown report with provider-specific breakdowns and test statistics.
"""
import json
import os
import sys
import subprocess
import sys
import xml.etree.ElementTree as ET
from collections import defaultdict
from datetime import datetime
from pathlib import Path
import json
from typing import Dict, List, Tuple, Optional
from typing import Dict, List, Optional, Tuple
# ANSI color codes for terminal output
class Colors:

3
backend/main.py
View file

@ -18,13 +18,12 @@ from litellm.proxy.db.db_url_settings import DatabaseURLSettings
DatabaseURLSettings.from_env().apply_to_env()
from litellm.proxy.proxy_server import app
from backend.routes.allowlist import (
BACKEND_EXACT_PATHS,
BACKEND_MOUNT_PATHS,
BACKEND_PATH_PREFIXES,
)
from litellm.proxy.proxy_server import app
def _is_backend_route(route) -> bool:

2
ci_cd/baseline_db.py
View file

@ -1,6 +1,6 @@
import subprocess
from pathlib import Path
from datetime import datetime
from pathlib import Path
def create_baseline():

2
ci_cd/check_files_match.py
View file

@ -1,6 +1,6 @@
import sys
import filecmp
import shutil
import sys
def main(argv=None):

1
ci_cd/run_migration.py
View file

@ -9,7 +9,6 @@ from pathlib import Path
import testing.postgresql
DESTRUCTIVE_PATTERN = re.compile(r"\bDROP\s+(COLUMN|TABLE|INDEX)\b", re.IGNORECASE)
DEFAULT_BASE_BRANCH = "litellm_internal_staging"

12
cookbook/Benchmarking_LLMs_by_use_case.ipynb vendored
View file

@ -94,10 +94,11 @@
},
"outputs": [],
"source": [
"from litellm import completion, completion_cost\n",
"import os\n",
"import time\n",
"\n",
"from litellm import completion, completion_cost\n",
"\n",
"# optional use litellm dashboard to view logs\n",
"# litellm.use_client = True\n",
"# litellm.token = \"ishaan_2@berri.ai\" # set your email\n",
@ -403,9 +404,10 @@
],
"source": [
"from IPython.core.interactiveshell import InteractiveShell\n",
"\n",
"InteractiveShell.ast_node_interactivity = \"all\"\n",
"from IPython.display import HTML\n",
"import pandas as pd\n",
"from IPython.display import HTML\n",
"\n",
"df = pd.DataFrame(data)\n",
"grouped_by_question = df.groupby('Question')\n",
@ -462,10 +464,11 @@
},
"outputs": [],
"source": [
"from litellm import completion, completion_cost\n",
"import os\n",
"import time\n",
"\n",
"from litellm import completion, completion_cost\n",
"\n",
"# optional use litellm dashboard to view logs\n",
"# litellm.use_client = True\n",
"# litellm.token = \"ishaan_2@berri.ai\" # set your email\n",
@ -723,9 +726,10 @@
],
"source": [
"from IPython.core.interactiveshell import InteractiveShell\n",
"\n",
"InteractiveShell.ast_node_interactivity = \"all\"\n",
"from IPython.display import HTML\n",
"import pandas as pd\n",
"from IPython.display import HTML\n",
"\n",
"df = pd.DataFrame(data_2)\n",
"grouped_by_question = df.groupby('Question')\n",

801
cookbook/Claude_(Anthropic)_with_Streaming_liteLLM_Examples.ipynb vendored
View file

@ -1,406 +1,407 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "ZwuaylskLxFu",
"outputId": "d684d6a3-32fe-4beb-c378-c39134bcf8cc"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Collecting litellm==0.1.363\n",
" Downloading litellm-0.1.363-py3-none-any.whl (34 kB)\n",
"Requirement already satisfied: openai<0.28.0,>=0.27.8 in /usr/local/lib/python3.10/dist-packages (from litellm==0.1.363) (0.27.8)\n",
"Requirement already satisfied: python-dotenv<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from litellm==0.1.363) (1.0.0)\n",
"Requirement already satisfied: tiktoken<0.5.0,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from litellm==0.1.363) (0.4.0)\n",
"Requirement already satisfied: requests>=2.20 in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.363) (2.31.0)\n",
"Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.363) (4.65.0)\n",
"Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.363) (3.8.5)\n",
"Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken<0.5.0,>=0.4.0->litellm==0.1.363) (2022.10.31)\n",
"Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (3.2.0)\n",
"Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (3.4)\n",
"Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.26.16)\n",
"Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (2023.7.22)\n",
"Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (23.1.0)\n",
"Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (6.0.4)\n",
"Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (4.0.2)\n",
"Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.9.2)\n",
"Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.4.0)\n",
"Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.3.1)\n",
"Installing collected packages: litellm\n",
" Attempting uninstall: litellm\n",
" Found existing installation: litellm 0.1.362\n",
" Uninstalling litellm-0.1.362:\n",
" Successfully uninstalled litellm-0.1.362\n",
"Successfully installed litellm-0.1.363\n"
]
}
],
"source": [
"!pip install litellm==\"0.1.363\""
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"id": "W216G__XL19Q"
},
"outputs": [],
"source": [
"# @title Import litellm & Set env variables\n",
"import litellm\n",
"import os\n",
"\n",
"os.environ[\"ANTHROPIC_API_KEY\"] = \" \" #@param"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "ff1lKwUMMLJj",
"outputId": "bfddf6f8-36d4-45e5-92dc-349083fa41b8"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"\n",
" Result from claude-instant-1 {'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': \" The Los Angeles Dodgers won the 2020 World Series, defeating the Tampa Bay Rays 4-2. It was the Dodgers' first World Series title since 1988.\"}}], 'created': 1691536677.2676156, 'model': 'claude-instant-1', 'usage': {'prompt_tokens': 30, 'completion_tokens': 32, 'total_tokens': 62}}\n",
"\n",
"\n",
" Result from claude-2 {'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': ' The Los Angeles Dodgers won'}}], 'created': 1691536677.944753, 'model': 'claude-2', 'usage': {'prompt_tokens': 30, 'completion_tokens': 5, 'total_tokens': 35}}\n"
]
}
],
"source": [
"# @title Request Claude Instant-1 and Claude-2\n",
"messages = [\n",
" {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
" {\"role\": \"user\", \"content\": \"Who won the world series in 2020?\"}\n",
" ]\n",
"\n",
"result = litellm.completion('claude-instant-1', messages)\n",
"print(\"\\n\\n Result from claude-instant-1\", result)\n",
"result = litellm.completion('claude-2', messages, max_tokens=5, temperature=0.2)\n",
"print(\"\\n\\n Result from claude-2\", result)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "06hWKnNQMrV-",
"outputId": "7fdec0eb-d4a9-4882-f9c4-987ff9a31114"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" Here\n",
"'s\n",
" a\n",
" quick\n",
" overview\n",
" of\n",
" how\n",
" a\n",
" court\n",
" case\n",
" can\n",
" reach\n",
" the\n",
" U\n",
".\n",
"S\n",
".\n",
" Supreme\n",
" Court\n",
":\n",
"\n",
"\n",
"-\n",
" The\n",
" case\n",
" must\n",
" first\n",
" be\n",
" heard\n",
" in\n",
" a\n",
" lower\n",
" trial\n",
" court\n",
" (\n",
"either\n",
" a\n",
" state\n",
" court\n",
" or\n",
" federal\n",
" district\n",
" court\n",
").\n",
" The\n",
" trial\n",
" court\n",
" makes\n",
" initial\n",
" r\n",
"ulings\n",
" and\n",
" produces\n",
" a\n",
" record\n",
" of\n",
" the\n",
" case\n",
".\n",
"\n",
"\n",
"-\n",
" The\n",
" losing\n",
" party\n",
" can\n",
" appeal\n",
" the\n",
" decision\n",
" to\n",
" an\n",
" appeals\n",
" court\n",
" (\n",
"a\n",
" state\n",
" appeals\n",
" court\n",
" for\n",
" state\n",
" cases\n",
",\n",
" or\n",
" a\n",
" federal\n",
" circuit\n",
" court\n",
" for\n",
" federal\n",
" cases\n",
").\n",
" The\n",
" appeals\n",
" court\n",
" reviews\n",
" the\n",
" trial\n",
" court\n",
"'s\n",
" r\n",
"ulings\n",
" and\n",
" can\n",
" affirm\n",
",\n",
" reverse\n",
",\n",
" or\n",
" modify\n",
" the\n",
" decision\n",
".\n",
"\n",
"\n",
"-\n",
" If\n",
" a\n",
" party\n",
" is\n",
" still\n",
" unsat\n",
"isf\n",
"ied\n",
" after\n",
" the\n",
" appeals\n",
" court\n",
" rules\n",
",\n",
" they\n",
" can\n",
" petition\n",
" the\n",
" Supreme\n",
" Court\n",
" to\n",
" hear\n",
" the\n",
" case\n",
" through\n",
" a\n",
" writ\n",
" of\n",
" cert\n",
"ior\n",
"ari\n",
".\n",
" \n",
"\n",
"\n",
"-\n",
" The\n",
" Supreme\n",
" Court\n",
" gets\n",
" thousands\n",
" of\n",
" cert\n",
" petitions\n",
" every\n",
" year\n",
" but\n",
" usually\n",
" only\n",
" agrees\n",
" to\n",
" hear\n",
" about\n",
" 100\n",
"-\n",
"150\n",
" of\n",
" cases\n",
" that\n",
" have\n",
" significant\n",
" national\n",
" importance\n",
" or\n",
" where\n",
" lower\n",
" courts\n",
" disagree\n",
" on\n",
" federal\n",
" law\n",
".\n",
" \n",
"\n",
"\n",
"-\n",
" If\n",
" 4\n",
" out\n",
" of\n",
" the\n",
" 9\n",
" Just\n",
"ices\n",
" vote\n",
" to\n",
" grant\n",
" cert\n",
" (\n",
"agree\n",
" to\n",
" hear\n",
" the\n",
" case\n",
"),\n",
" it\n",
" goes\n",
" on\n",
" the\n",
" Supreme\n",
" Court\n",
"'s\n",
" do\n",
"cket\n",
" for\n",
" arguments\n",
".\n",
"\n",
"\n",
"-\n",
" The\n",
" Supreme\n",
" Court\n",
" then\n",
" hears\n",
" oral\n",
" arguments\n",
",\n",
" considers\n",
" written\n",
" brief\n",
"s\n",
",\n",
" examines\n",
" the\n",
" lower\n",
" court\n",
" records\n",
",\n",
" and\n",
" issues\n",
" a\n",
" final\n",
" ruling\n",
" on\n",
" the\n",
" case\n",
",\n",
" which\n",
" serves\n",
" as\n",
" binding\n",
" precedent\n"
]
}
],
"source": [
"# @title Streaming Example: Request Claude-2\n",
"messages = [\n",
" {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
" {\"role\": \"user\", \"content\": \"how does a court case get to the Supreme Court?\"}\n",
" ]\n",
"\n",
"result = litellm.completion('claude-2', messages, stream=True)\n",
"for part in result:\n",
" print(part.choices[0].delta.content or \"\")\n",
"\n"
]
}
],
"metadata": {
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"colab": {
"provenance": []
"base_uri": "https://localhost:8080/"
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
"id": "ZwuaylskLxFu",
"outputId": "d684d6a3-32fe-4beb-c378-c39134bcf8cc"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Collecting litellm==0.1.363\n",
" Downloading litellm-0.1.363-py3-none-any.whl (34 kB)\n",
"Requirement already satisfied: openai<0.28.0,>=0.27.8 in /usr/local/lib/python3.10/dist-packages (from litellm==0.1.363) (0.27.8)\n",
"Requirement already satisfied: python-dotenv<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from litellm==0.1.363) (1.0.0)\n",
"Requirement already satisfied: tiktoken<0.5.0,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from litellm==0.1.363) (0.4.0)\n",
"Requirement already satisfied: requests>=2.20 in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.363) (2.31.0)\n",
"Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.363) (4.65.0)\n",
"Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.363) (3.8.5)\n",
"Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken<0.5.0,>=0.4.0->litellm==0.1.363) (2022.10.31)\n",
"Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (3.2.0)\n",
"Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (3.4)\n",
"Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.26.16)\n",
"Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (2023.7.22)\n",
"Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (23.1.0)\n",
"Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (6.0.4)\n",
"Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (4.0.2)\n",
"Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.9.2)\n",
"Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.4.0)\n",
"Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.3.1)\n",
"Installing collected packages: litellm\n",
" Attempting uninstall: litellm\n",
" Found existing installation: litellm 0.1.362\n",
" Uninstalling litellm-0.1.362:\n",
" Successfully uninstalled litellm-0.1.362\n",
"Successfully installed litellm-0.1.363\n"
]
}
],
"source": [
"!pip install litellm==\"0.1.363\""
]
},
"nbformat": 4,
"nbformat_minor": 0
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"id": "W216G__XL19Q"
},
"outputs": [],
"source": [
"# @title Import litellm & Set env variables\n",
"import os\n",
"\n",
"import litellm\n",
"\n",
"os.environ[\"ANTHROPIC_API_KEY\"] = \" \" #@param"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "ff1lKwUMMLJj",
"outputId": "bfddf6f8-36d4-45e5-92dc-349083fa41b8"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"\n",
" Result from claude-instant-1 {'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': \" The Los Angeles Dodgers won the 2020 World Series, defeating the Tampa Bay Rays 4-2. It was the Dodgers' first World Series title since 1988.\"}}], 'created': 1691536677.2676156, 'model': 'claude-instant-1', 'usage': {'prompt_tokens': 30, 'completion_tokens': 32, 'total_tokens': 62}}\n",
"\n",
"\n",
" Result from claude-2 {'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': ' The Los Angeles Dodgers won'}}], 'created': 1691536677.944753, 'model': 'claude-2', 'usage': {'prompt_tokens': 30, 'completion_tokens': 5, 'total_tokens': 35}}\n"
]
}
],
"source": [
"# @title Request Claude Instant-1 and Claude-2\n",
"messages = [\n",
" {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
" {\"role\": \"user\", \"content\": \"Who won the world series in 2020?\"}\n",
" ]\n",
"\n",
"result = litellm.completion('claude-instant-1', messages)\n",
"print(\"\\n\\n Result from claude-instant-1\", result)\n",
"result = litellm.completion('claude-2', messages, max_tokens=5, temperature=0.2)\n",
"print(\"\\n\\n Result from claude-2\", result)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "06hWKnNQMrV-",
"outputId": "7fdec0eb-d4a9-4882-f9c4-987ff9a31114"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" Here\n",
"'s\n",
" a\n",
" quick\n",
" overview\n",
" of\n",
" how\n",
" a\n",
" court\n",
" case\n",
" can\n",
" reach\n",
" the\n",
" U\n",
".\n",
"S\n",
".\n",
" Supreme\n",
" Court\n",
":\n",
"\n",
"\n",
"-\n",
" The\n",
" case\n",
" must\n",
" first\n",
" be\n",
" heard\n",
" in\n",
" a\n",
" lower\n",
" trial\n",
" court\n",
" (\n",
"either\n",
" a\n",
" state\n",
" court\n",
" or\n",
" federal\n",
" district\n",
" court\n",
").\n",
" The\n",
" trial\n",
" court\n",
" makes\n",
" initial\n",
" r\n",
"ulings\n",
" and\n",
" produces\n",
" a\n",
" record\n",
" of\n",
" the\n",
" case\n",
".\n",
"\n",
"\n",
"-\n",
" The\n",
" losing\n",
" party\n",
" can\n",
" appeal\n",
" the\n",
" decision\n",
" to\n",
" an\n",
" appeals\n",
" court\n",
" (\n",
"a\n",
" state\n",
" appeals\n",
" court\n",
" for\n",
" state\n",
" cases\n",
",\n",
" or\n",
" a\n",
" federal\n",
" circuit\n",
" court\n",
" for\n",
" federal\n",
" cases\n",
").\n",
" The\n",
" appeals\n",
" court\n",
" reviews\n",
" the\n",
" trial\n",
" court\n",
"'s\n",
" r\n",
"ulings\n",
" and\n",
" can\n",
" affirm\n",
",\n",
" reverse\n",
",\n",
" or\n",
" modify\n",
" the\n",
" decision\n",
".\n",
"\n",
"\n",
"-\n",
" If\n",
" a\n",
" party\n",
" is\n",
" still\n",
" unsat\n",
"isf\n",
"ied\n",
" after\n",
" the\n",
" appeals\n",
" court\n",
" rules\n",
",\n",
" they\n",
" can\n",
" petition\n",
" the\n",
" Supreme\n",
" Court\n",
" to\n",
" hear\n",
" the\n",
" case\n",
" through\n",
" a\n",
" writ\n",
" of\n",
" cert\n",
"ior\n",
"ari\n",
".\n",
" \n",
"\n",
"\n",
"-\n",
" The\n",
" Supreme\n",
" Court\n",
" gets\n",
" thousands\n",
" of\n",
" cert\n",
" petitions\n",
" every\n",
" year\n",
" but\n",
" usually\n",
" only\n",
" agrees\n",
" to\n",
" hear\n",
" about\n",
" 100\n",
"-\n",
"150\n",
" of\n",
" cases\n",
" that\n",
" have\n",
" significant\n",
" national\n",
" importance\n",
" or\n",
" where\n",
" lower\n",
" courts\n",
" disagree\n",
" on\n",
" federal\n",
" law\n",
".\n",
" \n",
"\n",
"\n",
"-\n",
" If\n",
" 4\n",
" out\n",
" of\n",
" the\n",
" 9\n",
" Just\n",
"ices\n",
" vote\n",
" to\n",
" grant\n",
" cert\n",
" (\n",
"agree\n",
" to\n",
" hear\n",
" the\n",
" case\n",
"),\n",
" it\n",
" goes\n",
" on\n",
" the\n",
" Supreme\n",
" Court\n",
"'s\n",
" do\n",
"cket\n",
" for\n",
" arguments\n",
".\n",
"\n",
"\n",
"-\n",
" The\n",
" Supreme\n",
" Court\n",
" then\n",
" hears\n",
" oral\n",
" arguments\n",
",\n",
" considers\n",
" written\n",
" brief\n",
"s\n",
",\n",
" examines\n",
" the\n",
" lower\n",
" court\n",
" records\n",
",\n",
" and\n",
" issues\n",
" a\n",
" final\n",
" ruling\n",
" on\n",
" the\n",
" case\n",
",\n",
" which\n",
" serves\n",
" as\n",
" binding\n",
" precedent\n"
]
}
],
"source": [
"# @title Streaming Example: Request Claude-2\n",
"messages = [\n",
" {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
" {\"role\": \"user\", \"content\": \"how does a court case get to the Supreme Court?\"}\n",
" ]\n",
"\n",
"result = litellm.completion('claude-2', messages, stream=True)\n",
"for part in result:\n",
" print(part.choices[0].delta.content or \"\")\n",
"\n"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

3
cookbook/Evaluating_LLMs.ipynb vendored
View file

@ -47,7 +47,8 @@
},
"outputs": [],
"source": [
"import os \n",
"import os\n",
"\n",
"os.environ[\"OPENAI_API_KEY\"] = \"...\"\n",
"os.environ[\"ANTHROPIC_API_KEY\"] = \"...\"\n",
"os.environ[\"REPLICATE_API_KEY\"] = \"...\""

7
cookbook/LiteLLM_Azure_and_OpenAI_example.ipynb vendored
View file

@ -159,6 +159,7 @@
"outputs": [],
"source": [
"import os\n",
"\n",
"from litellm import completion\n",
"\n",
"# openai configs\n",
@ -210,8 +211,10 @@
"source": [
"import os\n",
"import threading\n",
"\n",
"from litellm import completion\n",
"\n",
"\n",
"# Function to make a completion call\n",
"def make_completion(model, messages):\n",
" response = completion(\n",
@ -267,8 +270,10 @@
"source": [
"import os\n",
"import threading\n",
"\n",
"from litellm import completion\n",
"\n",
"\n",
"# Function to make a completion call\n",
"def make_completion(model, messages):\n",
" response = completion(\n",
@ -372,8 +377,10 @@
],
"source": [
"import os\n",
"\n",
"from litellm import completion\n",
"\n",
"\n",
"# Function to make both OpenAI and Azure completions\n",
"def make_completions():\n",
" # Set your OpenAI API key\n",

615
cookbook/LiteLLM_Bedrock.ipynb vendored
View file

@ -1,310 +1,311 @@
{
"cells": [
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "fNkMBurtxawJ"
},
"source": [
"# LiteLLM Bedrock Usage\n",
"Important Note: For Bedrock Requests you need to ensure you have `pip install boto3>=1.28.57`, boto3 supports bedrock from `boto3>=1.28.57` and higher "
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "htAufI28xeSy"
},
"source": [
"## Pre-Requisites"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "jT5GbPjAuDTp"
},
"outputs": [],
"source": [
"!pip install litellm\n",
"!pip install boto3>=1.28.57 # this version onwards has bedrock support"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "H4Vu4er2xnfI"
},
"source": [
"## Set Bedrock/AWS Credentials"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"id": "CtTrBthWxp-t"
},
"outputs": [],
"source": [
"import os\n",
"os.environ[\"AWS_ACCESS_KEY_ID\"] = \"\" # Access key\n",
"os.environ[\"AWS_SECRET_ACCESS_KEY\"] = \"\" # Secret access key\n",
"os.environ[\"AWS_REGION_NAME\"] = \"\""
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "ycRK9NUdx1EI"
},
"source": [
"## Anthropic Requests"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "tgkuoHa5uLOy",
"outputId": "27a78e86-c6a7-4bcc-8559-0813cb978426"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Claude instant 1, response\n",
"{\n",
" \"object\": \"chat.completion\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \" I'm doing well, thanks for asking!\",\n",
" \"role\": \"assistant\",\n",
" \"logprobs\": null\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-4f2e64a1-56d2-43f2-90d3-60ffd6f5086d\",\n",
" \"created\": 1696256761.3265705,\n",
" \"model\": \"anthropic.claude-instant-v1\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": 11,\n",
" \"completion_tokens\": 9,\n",
" \"total_tokens\": 20\n",
" },\n",
" \"finish_reason\": \"stop_sequence\"\n",
"}\n",
"Claude v2, response\n",
"{\n",
" \"object\": \"chat.completion\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \" I'm doing well, thanks for asking!\",\n",
" \"role\": \"assistant\",\n",
" \"logprobs\": null\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-34f59b33-f94e-40c2-8bdb-f4af0813405e\",\n",
" \"created\": 1696256762.2137017,\n",
" \"model\": \"anthropic.claude-v2\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": 11,\n",
" \"completion_tokens\": 9,\n",
" \"total_tokens\": 20\n",
" },\n",
" \"finish_reason\": \"stop_sequence\"\n",
"}\n"
]
}
],
"source": [
"from litellm import completion\n",
"\n",
"response = completion(\n",
" model=\"bedrock/anthropic.claude-instant-v1\",\n",
" messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
")\n",
"print(\"Claude instant 1, response\")\n",
"print(response)\n",
"\n",
"\n",
"response = completion(\n",
" model=\"bedrock/anthropic.claude-v2\",\n",
" messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
")\n",
"print(\"Claude v2, response\")\n",
"print(response)"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "HnM-HtM3yFMT"
},
"source": [
"## Anthropic Requests - With Streaming"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "_JZvg2yovRsU"
},
"outputs": [],
"source": [
"from litellm import completion\n",
"\n",
"response = completion(\n",
" model=\"bedrock/anthropic.claude-instant-v1\",\n",
" messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
" stream=True,\n",
")\n",
"print(\"Claude instant 1, response\")\n",
"for chunk in response:\n",
" print(chunk)\n",
"\n",
"\n",
"response = completion(\n",
" model=\"bedrock/anthropic.claude-v2\",\n",
" messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
" stream=True\n",
")\n",
"print(\"Claude v2, response\")\n",
"print(response)\n",
"for chunk in response:\n",
" print(chunk)"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "zj1U1mh9zEhP"
},
"source": [
"## A121 Requests"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "6wK6MZLovU7r",
"outputId": "4cf80c04-f15d-4066-b4c7-113b551538de"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"J2 ultra response\n",
"{\n",
" \"object\": \"chat.completion\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \"\\nHi, I'm doing well, thanks for asking! How about you?\",\n",
" \"role\": \"assistant\",\n",
" \"logprobs\": null\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-f2de678f-0e70-4e36-a01f-8b184c2e4d50\",\n",
" \"created\": 1696257116.044311,\n",
" \"model\": \"ai21.j2-ultra\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": 6,\n",
" \"completion_tokens\": 16,\n",
" \"total_tokens\": 22\n",
" }\n",
"}\n",
"J2 mid response\n",
"{\n",
" \"object\": \"chat.completion\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \"\\nGood. And you?\",\n",
" \"role\": \"assistant\",\n",
" \"logprobs\": null\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-420d6bf9-36d8-484b-93b4-4c9e00f7ce2e\",\n",
" \"created\": 1696257116.5756805,\n",
" \"model\": \"ai21.j2-mid\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": 6,\n",
" \"completion_tokens\": 6,\n",
" \"total_tokens\": 12\n",
" }\n",
"}\n"
]
}
],
"source": [
"response = completion(\n",
" model=\"bedrock/ai21.j2-ultra\",\n",
" messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
")\n",
"print(\"J2 ultra response\")\n",
"print(response)\n",
"\n",
"response = completion(\n",
" model=\"bedrock/ai21.j2-mid\",\n",
" messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
")\n",
"print(\"J2 mid response\")\n",
"print(response)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "Y5gGZIwzzSON"
},
"outputs": [],
"source": []
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
"cells": [
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "fNkMBurtxawJ"
},
"source": [
"# LiteLLM Bedrock Usage\n",
"Important Note: For Bedrock Requests you need to ensure you have `pip install boto3>=1.28.57`, boto3 supports bedrock from `boto3>=1.28.57` and higher "
]
},
"nbformat": 4,
"nbformat_minor": 0
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "htAufI28xeSy"
},
"source": [
"## Pre-Requisites"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "jT5GbPjAuDTp"
},
"outputs": [],
"source": [
"!pip install litellm\n",
"!pip install boto3>=1.28.57 # this version onwards has bedrock support"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "H4Vu4er2xnfI"
},
"source": [
"## Set Bedrock/AWS Credentials"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"id": "CtTrBthWxp-t"
},
"outputs": [],
"source": [
"import os\n",
"\n",
"os.environ[\"AWS_ACCESS_KEY_ID\"] = \"\" # Access key\n",
"os.environ[\"AWS_SECRET_ACCESS_KEY\"] = \"\" # Secret access key\n",
"os.environ[\"AWS_REGION_NAME\"] = \"\""
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "ycRK9NUdx1EI"
},
"source": [
"## Anthropic Requests"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "tgkuoHa5uLOy",
"outputId": "27a78e86-c6a7-4bcc-8559-0813cb978426"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Claude instant 1, response\n",
"{\n",
" \"object\": \"chat.completion\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \" I'm doing well, thanks for asking!\",\n",
" \"role\": \"assistant\",\n",
" \"logprobs\": null\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-4f2e64a1-56d2-43f2-90d3-60ffd6f5086d\",\n",
" \"created\": 1696256761.3265705,\n",
" \"model\": \"anthropic.claude-instant-v1\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": 11,\n",
" \"completion_tokens\": 9,\n",
" \"total_tokens\": 20\n",
" },\n",
" \"finish_reason\": \"stop_sequence\"\n",
"}\n",
"Claude v2, response\n",
"{\n",
" \"object\": \"chat.completion\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \" I'm doing well, thanks for asking!\",\n",
" \"role\": \"assistant\",\n",
" \"logprobs\": null\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-34f59b33-f94e-40c2-8bdb-f4af0813405e\",\n",
" \"created\": 1696256762.2137017,\n",
" \"model\": \"anthropic.claude-v2\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": 11,\n",
" \"completion_tokens\": 9,\n",
" \"total_tokens\": 20\n",
" },\n",
" \"finish_reason\": \"stop_sequence\"\n",
"}\n"
]
}
],
"source": [
"from litellm import completion\n",
"\n",
"response = completion(\n",
" model=\"bedrock/anthropic.claude-instant-v1\",\n",
" messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
")\n",
"print(\"Claude instant 1, response\")\n",
"print(response)\n",
"\n",
"\n",
"response = completion(\n",
" model=\"bedrock/anthropic.claude-v2\",\n",
" messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
")\n",
"print(\"Claude v2, response\")\n",
"print(response)"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "HnM-HtM3yFMT"
},
"source": [
"## Anthropic Requests - With Streaming"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "_JZvg2yovRsU"
},
"outputs": [],
"source": [
"from litellm import completion\n",
"\n",
"response = completion(\n",
" model=\"bedrock/anthropic.claude-instant-v1\",\n",
" messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
" stream=True,\n",
")\n",
"print(\"Claude instant 1, response\")\n",
"for chunk in response:\n",
" print(chunk)\n",
"\n",
"\n",
"response = completion(\n",
" model=\"bedrock/anthropic.claude-v2\",\n",
" messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
" stream=True\n",
")\n",
"print(\"Claude v2, response\")\n",
"print(response)\n",
"for chunk in response:\n",
" print(chunk)"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "zj1U1mh9zEhP"
},
"source": [
"## A121 Requests"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "6wK6MZLovU7r",
"outputId": "4cf80c04-f15d-4066-b4c7-113b551538de"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"J2 ultra response\n",
"{\n",
" \"object\": \"chat.completion\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \"\\nHi, I'm doing well, thanks for asking! How about you?\",\n",
" \"role\": \"assistant\",\n",
" \"logprobs\": null\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-f2de678f-0e70-4e36-a01f-8b184c2e4d50\",\n",
" \"created\": 1696257116.044311,\n",
" \"model\": \"ai21.j2-ultra\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": 6,\n",
" \"completion_tokens\": 16,\n",
" \"total_tokens\": 22\n",
" }\n",
"}\n",
"J2 mid response\n",
"{\n",
" \"object\": \"chat.completion\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \"\\nGood. And you?\",\n",
" \"role\": \"assistant\",\n",
" \"logprobs\": null\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-420d6bf9-36d8-484b-93b4-4c9e00f7ce2e\",\n",
" \"created\": 1696257116.5756805,\n",
" \"model\": \"ai21.j2-mid\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": 6,\n",
" \"completion_tokens\": 6,\n",
" \"total_tokens\": 12\n",
" }\n",
"}\n"
]
}
],
"source": [
"response = completion(\n",
" model=\"bedrock/ai21.j2-ultra\",\n",
" messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
")\n",
"print(\"J2 ultra response\")\n",
"print(response)\n",
"\n",
"response = completion(\n",
" model=\"bedrock/ai21.j2-mid\",\n",
" messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
")\n",
"print(\"J2 mid response\")\n",
"print(response)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "Y5gGZIwzzSON"
},
"outputs": [],
"source": []
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

11
cookbook/LiteLLM_CometAPI.ipynb vendored
View file

@ -119,6 +119,7 @@
],
"source": [
"from litellm import completion\n",
"\n",
"response = completion(\n",
" model=\"cometapi/claude-sonnet-4-5-20250929\",\n",
" messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
@ -256,9 +257,11 @@
}
],
"source": [
"from litellm import acompletion\n",
"import asyncio\n",
"\n",
"from litellm import acompletion\n",
"\n",
"\n",
"async def test_get_response():\n",
" user_message = \"Hello, how are you?\"\n",
" messages = [{\"content\": user_message, \"role\": \"user\"}]\n",
@ -313,8 +316,12 @@
}
],
"source": [
"import asyncio\n",
"import os\n",
"import traceback\n",
"\n",
"from litellm import acompletion\n",
"import asyncio, os, traceback\n",
"\n",
"\n",
"async def completion_call():\n",
" try:\n",

1
cookbook/LiteLLM_Comparing_LLMs.ipynb vendored
View file

@ -58,6 +58,7 @@
"outputs": [],
"source": [
"import os\n",
"\n",
"os.environ['OPENAI_API_KEY'] = \"\"\n",
"os.environ['ANTHROPIC_API_KEY'] = \"\""
]

476
cookbook/LiteLLM_Completion_Cost.ipynb vendored
View file

@ -1,241 +1,247 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "BgWr0PsUR3vV"
},
"source": [
"# Use LiteLLM to calculate costs for all your completion calls\n",
"In this notebook we'll use `litellm.completion_cost` to get completion costs"
]
},
"cells": [
{
"cell_type": "markdown",
"source": [
"# Use LiteLLM to calculate costs for all your completion calls\n",
"In this notebook we'll use `litellm.completion_cost` to get completion costs"
],
"metadata": {
"id": "BgWr0PsUR3vV"
}
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "ViczFTjsDzSI"
},
"outputs": [],
"source": [
"!pip install litellm==0.1.549 # use 0.1.549 or later"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "k_1CWUwmSNtj"
},
"source": [
"## Calculating costs for gpt-3.5 turbo completion()"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "Tp0fyk-jD0pP",
"outputId": "ce885fb3-3237-41b2-9d8b-3fb30bba498b"
},
"outputs": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "ViczFTjsDzSI"
},
"outputs": [],
"source": [
"!pip install litellm==0.1.549 # use 0.1.549 or later"
]
},
{
"cell_type": "markdown",
"source": [
"## Calculating costs for gpt-3.5 turbo completion()"
],
"metadata": {
"id": "k_1CWUwmSNtj"
}
},
{
"cell_type": "code",
"source": [
"from litellm import completion, completion_cost\n",
"import os\n",
"os.environ['OPENAI_API_KEY'] = \"\"\n",
"\n",
"messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
"response = completion(\n",
" model=\"gpt-3.5-turbo\",\n",
" messages=messages,\n",
")\n",
"\n",
"print(response)\n",
"\n",
"cost = completion_cost(completion_response=response)\n",
"formatted_string = f\"Cost for completion call: ${float(cost):.10f}\"\n",
"print(formatted_string)\n"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "Tp0fyk-jD0pP",
"outputId": "ce885fb3-3237-41b2-9d8b-3fb30bba498b"
},
"execution_count": 6,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"got response\n",
"{\n",
" \"id\": \"chatcmpl-7vyCApIZaCxP36kb9meUMN2DFSJPh\",\n",
" \"object\": \"chat.completion\",\n",
" \"created\": 1694050442,\n",
" \"model\": \"gpt-3.5-turbo-0613\",\n",
" \"choices\": [\n",
" {\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": \"Hello! I'm an AI and I don't have feelings, but I'm here to help you. How can I assist you today?\"\n",
" },\n",
" \"finish_reason\": \"stop\"\n",
" }\n",
" ],\n",
" \"usage\": {\n",
" \"prompt_tokens\": 13,\n",
" \"completion_tokens\": 28,\n",
" \"total_tokens\": 41\n",
" }\n",
"}\n",
"Cost for completion call: $0.0000755000\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"## Calculating costs for Together Computer completion()"
],
"metadata": {
"id": "AjDs4G-uS6PS"
}
},
{
"cell_type": "code",
"source": [
"from litellm import completion, completion_cost\n",
"import os\n",
"os.environ['TOGETHERAI_API_KEY'] = \"\"\n",
"\n",
"messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
"response = completion(\n",
" model=\"togethercomputer/llama-2-70b-chat\",\n",
" messages=messages,\n",
")\n",
"\n",
"print(response)\n",
"\n",
"cost = completion_cost(completion_response=response)\n",
"formatted_string = f\"Cost for completion call: ${float(cost):.10f}\"\n",
"print(formatted_string)\n"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "jMPsUV-KEa6a",
"outputId": "7a69b291-f149-4b9c-8a78-9c8142bac759"
},
"execution_count": 7,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"{\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \"Hello! I'm doing well, thanks for asking. I hope you're having a great\",\n",
" \"role\": \"assistant\",\n",
" \"logprobs\": null\n",
" }\n",
" }\n",
" ],\n",
" \"created\": 1694050771.2821715,\n",
" \"model\": \"togethercomputer/llama-2-70b-chat\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": 12,\n",
" \"completion_tokens\": 18,\n",
" \"total_tokens\": 30\n",
" }\n",
"}\n",
"Cost for completion call: $0.0000900000\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"## Calculating costs for Replicate Llama2 completion()"
],
"metadata": {
"id": "vEa4s6-7TANS"
}
},
{
"cell_type": "code",
"source": [
"from litellm import completion, completion_cost\n",
"import os\n",
"os.environ['REPLICATE_API_KEY'] = \"\"\n",
"\n",
"messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
"response = completion(\n",
" model=\"replicate/llama-2-70b-chat:2796ee9483c3fd7aa2e171d38f4ca12251a30609463dcfd4cd76703f22e96cdf\",\n",
" messages=messages,\n",
")\n",
"\n",
"print(response)\n",
"\n",
"cost = completion_cost(completion_response=response)\n",
"formatted_string = f\"Cost for completion call: ${float(cost):.10f}\"\n",
"print(formatted_string)\n"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "Xf1TKRDuS1bR",
"outputId": "cfb2b484-a6e5-41ad-86c5-7e66aba27648"
},
"execution_count": 8,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"{\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \" Hello! I'm doing well, thanks for asking. How about you? Is there anything you need help with today?\",\n",
" \"role\": \"assistant\",\n",
" \"logprobs\": null\n",
" }\n",
" }\n",
" ],\n",
" \"created\": 1694050893.4534576,\n",
" \"model\": \"replicate/llama-2-70b-chat:2796ee9483c3fd7aa2e171d38f4ca12251a30609463dcfd4cd76703f22e96cdf\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": 6,\n",
" \"completion_tokens\": 24,\n",
" \"total_tokens\": 30\n",
" },\n",
" \"ended\": 1694050896.6689413\n",
"}\n",
"total_replicate_run_time 3.2154836654663086\n",
"Cost for completion call: $0.0045016771\n"
]
}
]
"name": "stdout",
"output_type": "stream",
"text": [
"got response\n",
"{\n",
" \"id\": \"chatcmpl-7vyCApIZaCxP36kb9meUMN2DFSJPh\",\n",
" \"object\": \"chat.completion\",\n",
" \"created\": 1694050442,\n",
" \"model\": \"gpt-3.5-turbo-0613\",\n",
" \"choices\": [\n",
" {\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": \"Hello! I'm an AI and I don't have feelings, but I'm here to help you. How can I assist you today?\"\n",
" },\n",
" \"finish_reason\": \"stop\"\n",
" }\n",
" ],\n",
" \"usage\": {\n",
" \"prompt_tokens\": 13,\n",
" \"completion_tokens\": 28,\n",
" \"total_tokens\": 41\n",
" }\n",
"}\n",
"Cost for completion call: $0.0000755000\n"
]
}
]
],
"source": [
"import os\n",
"\n",
"from litellm import completion, completion_cost\n",
"\n",
"os.environ['OPENAI_API_KEY'] = \"\"\n",
"\n",
"messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
"response = completion(\n",
" model=\"gpt-3.5-turbo\",\n",
" messages=messages,\n",
")\n",
"\n",
"print(response)\n",
"\n",
"cost = completion_cost(completion_response=response)\n",
"formatted_string = f\"Cost for completion call: ${float(cost):.10f}\"\n",
"print(formatted_string)\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "AjDs4G-uS6PS"
},
"source": [
"## Calculating costs for Together Computer completion()"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "jMPsUV-KEa6a",
"outputId": "7a69b291-f149-4b9c-8a78-9c8142bac759"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \"Hello! I'm doing well, thanks for asking. I hope you're having a great\",\n",
" \"role\": \"assistant\",\n",
" \"logprobs\": null\n",
" }\n",
" }\n",
" ],\n",
" \"created\": 1694050771.2821715,\n",
" \"model\": \"togethercomputer/llama-2-70b-chat\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": 12,\n",
" \"completion_tokens\": 18,\n",
" \"total_tokens\": 30\n",
" }\n",
"}\n",
"Cost for completion call: $0.0000900000\n"
]
}
],
"source": [
"import os\n",
"\n",
"from litellm import completion, completion_cost\n",
"\n",
"os.environ['TOGETHERAI_API_KEY'] = \"\"\n",
"\n",
"messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
"response = completion(\n",
" model=\"togethercomputer/llama-2-70b-chat\",\n",
" messages=messages,\n",
")\n",
"\n",
"print(response)\n",
"\n",
"cost = completion_cost(completion_response=response)\n",
"formatted_string = f\"Cost for completion call: ${float(cost):.10f}\"\n",
"print(formatted_string)\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "vEa4s6-7TANS"
},
"source": [
"## Calculating costs for Replicate Llama2 completion()"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "Xf1TKRDuS1bR",
"outputId": "cfb2b484-a6e5-41ad-86c5-7e66aba27648"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \" Hello! I'm doing well, thanks for asking. How about you? Is there anything you need help with today?\",\n",
" \"role\": \"assistant\",\n",
" \"logprobs\": null\n",
" }\n",
" }\n",
" ],\n",
" \"created\": 1694050893.4534576,\n",
" \"model\": \"replicate/llama-2-70b-chat:2796ee9483c3fd7aa2e171d38f4ca12251a30609463dcfd4cd76703f22e96cdf\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": 6,\n",
" \"completion_tokens\": 24,\n",
" \"total_tokens\": 30\n",
" },\n",
" \"ended\": 1694050896.6689413\n",
"}\n",
"total_replicate_run_time 3.2154836654663086\n",
"Cost for completion call: $0.0045016771\n"
]
}
],
"source": [
"import os\n",
"\n",
"from litellm import completion, completion_cost\n",
"\n",
"os.environ['REPLICATE_API_KEY'] = \"\"\n",
"\n",
"messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
"response = completion(\n",
" model=\"replicate/llama-2-70b-chat:2796ee9483c3fd7aa2e171d38f4ca12251a30609463dcfd4cd76703f22e96cdf\",\n",
" messages=messages,\n",
")\n",
"\n",
"print(response)\n",
"\n",
"cost = completion_cost(completion_response=response)\n",
"formatted_string = f\"Cost for completion call: ${float(cost):.10f}\"\n",
"print(formatted_string)\n"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

500
cookbook/LiteLLM_HuggingFace.ipynb vendored
View file

@ -1,252 +1,254 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "9dKM5k8qsMIj"
},
"source": [
"## LiteLLM Hugging Face\n",
"\n",
"Docs for huggingface: https://docs.litellm.ai/docs/providers/huggingface\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "BVDdmCp-o97j"
},
"outputs": [],
"source": [
"!pip install litellm"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "yp5UXRqtpu9f"
},
"source": [
"## Serverless Inference Providers\n",
"\n",
"Read more about Inference Providers here: https://huggingface.co/blog/inference-providers.\n",
"\n",
"In order to use litellm with Hugging Face Inference Providers, you need to set `model=huggingface/<provider>/<model-id>`.\n",
"\n",
"Example: `huggingface/together/deepseek-ai/DeepSeek-R1` to run DeepSeek-R1 (https://huggingface.co/deepseek-ai/DeepSeek-R1) through Together AI.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "Pi5Oww8gpCUm",
"outputId": "659a67c7-f90d-4c06-b94e-2c4aa92d897a"
},
"outputs": [],
"source": [
"import os\n",
"from litellm import completion\n",
"\n",
"# You can create a HF token here: https://huggingface.co/settings/tokens\n",
"os.environ[\"HF_TOKEN\"] = \"hf_xxxxxx\"\n",
"\n",
"# Call DeepSeek-R1 model through Together AI\n",
"response = completion(\n",
" model=\"huggingface/together/deepseek-ai/DeepSeek-R1\",\n",
" messages=[{\"content\": \"How many r's are in the word `strawberry`?\", \"role\": \"user\"}],\n",
")\n",
"print(response)"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "EU0UubrKzTFe"
},
"source": [
"## Streaming\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "y-QfIvA-uJKX",
"outputId": "b007bb98-00d0-44a4-8264-c8a2caed6768"
},
"outputs": [],
"source": [
"import os\n",
"from litellm import completion\n",
"\n",
"os.environ[\"HF_TOKEN\"] = \"hf_xxxxxx\"\n",
"\n",
"response = completion(\n",
" model=\"huggingface/together/deepseek-ai/DeepSeek-R1\",\n",
" messages=[\n",
" {\n",
" \"role\": \"user\",\n",
" \"content\": \"How many r's are in the word `strawberry`?\",\n",
" \n",
" }\n",
" ],\n",
" stream=True,\n",
")\n",
"\n",
"for chunk in response:\n",
" print(chunk)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## With images as input\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from litellm import completion\n",
"\n",
"# Set your Hugging Face Token\n",
"os.environ[\"HF_TOKEN\"] = \"hf_xxxxxx\"\n",
"\n",
"messages = [\n",
" {\n",
" \"role\": \"user\",\n",
" \"content\": [\n",
" {\"type\": \"text\", \"text\": \"What's in this image?\"},\n",
" {\n",
" \"type\": \"image_url\",\n",
" \"image_url\": {\n",
" \"url\": \"https://awsmp-logos.s3.amazonaws.com/seller-xw5kijmvmzasy/c233c9ade2ccb5491072ae232c814942.png\",\n",
" },\n",
" },\n",
" ],\n",
" }\n",
"]\n",
"\n",
"response = completion(\n",
" model=\"huggingface/sambanova/meta-llama/Llama-3.3-70B-Instruct\",\n",
" messages=messages,\n",
")\n",
"print(response.choices[0])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Tools - Function Calling\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"from litellm import completion\n",
"\n",
"\n",
"# Set your Hugging Face Token\n",
"os.environ[\"HF_TOKEN\"] = \"hf_xxxxxx\"\n",
"\n",
"tools = [\n",
" {\n",
" \"type\": \"function\",\n",
" \"function\": {\n",
" \"name\": \"get_current_weather\",\n",
" \"description\": \"Get the current weather in a given location\",\n",
" \"parameters\": {\n",
" \"type\": \"object\",\n",
" \"properties\": {\n",
" \"location\": {\n",
" \"type\": \"string\",\n",
" \"description\": \"The city and state, e.g. San Francisco, CA\",\n",
" },\n",
" \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"]},\n",
" },\n",
" \"required\": [\"location\"],\n",
" },\n",
" },\n",
" }\n",
"]\n",
"messages = [{\"role\": \"user\", \"content\": \"What's the weather like in Boston today?\"}]\n",
"\n",
"response = completion(\n",
" model=\"huggingface/sambanova/meta-llama/Llama-3.1-8B-Instruct\", messages=messages, tools=tools, tool_choice=\"auto\"\n",
")\n",
"print(response)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Hugging Face Dedicated Inference Endpoints\n",
"\n",
"Steps to use\n",
"\n",
"- Create your own Hugging Face dedicated endpoint here: https://ui.endpoints.huggingface.co/\n",
"- Set `api_base` to your deployed api base\n",
"- set the model to `huggingface/tgi` so that litellm knows it's a huggingface Deployed Inference Endpoint.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import litellm\n",
"\n",
"\n",
"response = litellm.completion(\n",
" model=\"huggingface/tgi\",\n",
" messages=[{\"content\": \"Hello, how are you?\", \"role\": \"user\"}],\n",
" api_base=\"https://my-endpoint.endpoints.huggingface.cloud/v1/\",\n",
")\n",
"print(response)"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": ".venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.0"
}
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "9dKM5k8qsMIj"
},
"source": [
"## LiteLLM Hugging Face\n",
"\n",
"Docs for huggingface: https://docs.litellm.ai/docs/providers/huggingface\n"
]
},
"nbformat": 4,
"nbformat_minor": 0
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "BVDdmCp-o97j"
},
"outputs": [],
"source": [
"!pip install litellm"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "yp5UXRqtpu9f"
},
"source": [
"## Serverless Inference Providers\n",
"\n",
"Read more about Inference Providers here: https://huggingface.co/blog/inference-providers.\n",
"\n",
"In order to use litellm with Hugging Face Inference Providers, you need to set `model=huggingface/<provider>/<model-id>`.\n",
"\n",
"Example: `huggingface/together/deepseek-ai/DeepSeek-R1` to run DeepSeek-R1 (https://huggingface.co/deepseek-ai/DeepSeek-R1) through Together AI.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "Pi5Oww8gpCUm",
"outputId": "659a67c7-f90d-4c06-b94e-2c4aa92d897a"
},
"outputs": [],
"source": [
"import os\n",
"\n",
"from litellm import completion\n",
"\n",
"# You can create a HF token here: https://huggingface.co/settings/tokens\n",
"os.environ[\"HF_TOKEN\"] = \"hf_xxxxxx\"\n",
"\n",
"# Call DeepSeek-R1 model through Together AI\n",
"response = completion(\n",
" model=\"huggingface/together/deepseek-ai/DeepSeek-R1\",\n",
" messages=[{\"content\": \"How many r's are in the word `strawberry`?\", \"role\": \"user\"}],\n",
")\n",
"print(response)"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "EU0UubrKzTFe"
},
"source": [
"## Streaming\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "y-QfIvA-uJKX",
"outputId": "b007bb98-00d0-44a4-8264-c8a2caed6768"
},
"outputs": [],
"source": [
"import os\n",
"\n",
"from litellm import completion\n",
"\n",
"os.environ[\"HF_TOKEN\"] = \"hf_xxxxxx\"\n",
"\n",
"response = completion(\n",
" model=\"huggingface/together/deepseek-ai/DeepSeek-R1\",\n",
" messages=[\n",
" {\n",
" \"role\": \"user\",\n",
" \"content\": \"How many r's are in the word `strawberry`?\",\n",
" \n",
" }\n",
" ],\n",
" stream=True,\n",
")\n",
"\n",
"for chunk in response:\n",
" print(chunk)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## With images as input\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from litellm import completion\n",
"\n",
"# Set your Hugging Face Token\n",
"os.environ[\"HF_TOKEN\"] = \"hf_xxxxxx\"\n",
"\n",
"messages = [\n",
" {\n",
" \"role\": \"user\",\n",
" \"content\": [\n",
" {\"type\": \"text\", \"text\": \"What's in this image?\"},\n",
" {\n",
" \"type\": \"image_url\",\n",
" \"image_url\": {\n",
" \"url\": \"https://awsmp-logos.s3.amazonaws.com/seller-xw5kijmvmzasy/c233c9ade2ccb5491072ae232c814942.png\",\n",
" },\n",
" },\n",
" ],\n",
" }\n",
"]\n",
"\n",
"response = completion(\n",
" model=\"huggingface/sambanova/meta-llama/Llama-3.3-70B-Instruct\",\n",
" messages=messages,\n",
")\n",
"print(response.choices[0])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Tools - Function Calling\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"\n",
"from litellm import completion\n",
"\n",
"# Set your Hugging Face Token\n",
"os.environ[\"HF_TOKEN\"] = \"hf_xxxxxx\"\n",
"\n",
"tools = [\n",
" {\n",
" \"type\": \"function\",\n",
" \"function\": {\n",
" \"name\": \"get_current_weather\",\n",
" \"description\": \"Get the current weather in a given location\",\n",
" \"parameters\": {\n",
" \"type\": \"object\",\n",
" \"properties\": {\n",
" \"location\": {\n",
" \"type\": \"string\",\n",
" \"description\": \"The city and state, e.g. San Francisco, CA\",\n",
" },\n",
" \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"]},\n",
" },\n",
" \"required\": [\"location\"],\n",
" },\n",
" },\n",
" }\n",
"]\n",
"messages = [{\"role\": \"user\", \"content\": \"What's the weather like in Boston today?\"}]\n",
"\n",
"response = completion(\n",
" model=\"huggingface/sambanova/meta-llama/Llama-3.1-8B-Instruct\", messages=messages, tools=tools, tool_choice=\"auto\"\n",
")\n",
"print(response)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Hugging Face Dedicated Inference Endpoints\n",
"\n",
"Steps to use\n",
"\n",
"- Create your own Hugging Face dedicated endpoint here: https://ui.endpoints.huggingface.co/\n",
"- Set `api_base` to your deployed api base\n",
"- set the model to `huggingface/tgi` so that litellm knows it's a huggingface Deployed Inference Endpoint.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"\n",
"import litellm\n",
"\n",
"response = litellm.completion(\n",
" model=\"huggingface/tgi\",\n",
" messages=[{\"content\": \"Hello, how are you?\", \"role\": \"user\"}],\n",
" api_base=\"https://my-endpoint.endpoints.huggingface.cloud/v1/\",\n",
")\n",
"print(response)"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": ".venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.0"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

189
cookbook/LiteLLM_NovitaAI_Cookbook.ipynb vendored
View file

@ -1,97 +1,98 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "iFEmsVJI_2BR"
},
"source": [
"# LiteLLM NovitaAI Cookbook"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "cBlUhCEP_xj4"
},
"outputs": [],
"source": [
"!pip install litellm"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "p-MQqWOT_1a7"
},
"outputs": [],
"source": [
"import os\n",
"\n",
"os.environ['NOVITA_API_KEY'] = \"\""
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "Ze8JqMqWAARO"
},
"outputs": [],
"source": [
"from litellm import completion\n",
"response = completion(\n",
" model=\"novita/deepseek/deepseek-r1\",\n",
" messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
")\n",
"response"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "-LnhELrnAM_J"
},
"outputs": [],
"source": [
"response = completion(\n",
" model=\"novita/deepseek/deepseek-r1\",\n",
" messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
")\n",
"response"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "dJBOUYdwCEn1"
},
"outputs": [],
"source": [
"response = completion(\n",
" model=\"mistralai/mistral-7b-instruct\",\n",
" messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
")\n",
"response"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "iFEmsVJI_2BR"
},
"source": [
"# LiteLLM NovitaAI Cookbook"
]
},
"nbformat": 4,
"nbformat_minor": 0
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "cBlUhCEP_xj4"
},
"outputs": [],
"source": [
"!pip install litellm"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "p-MQqWOT_1a7"
},
"outputs": [],
"source": [
"import os\n",
"\n",
"os.environ['NOVITA_API_KEY'] = \"\""
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "Ze8JqMqWAARO"
},
"outputs": [],
"source": [
"from litellm import completion\n",
"\n",
"response = completion(\n",
" model=\"novita/deepseek/deepseek-r1\",\n",
" messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
")\n",
"response"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "-LnhELrnAM_J"
},
"outputs": [],
"source": [
"response = completion(\n",
" model=\"novita/deepseek/deepseek-r1\",\n",
" messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
")\n",
"response"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "dJBOUYdwCEn1"
},
"outputs": [],
"source": [
"response = completion(\n",
" model=\"mistralai/mistral-7b-instruct\",\n",
" messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
")\n",
"response"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

353
cookbook/LiteLLM_OpenRouter.ipynb vendored
View file

@ -1,179 +1,180 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "iFEmsVJI_2BR"
},
"source": [
"# LiteLLM OpenRouter Cookbook"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "cBlUhCEP_xj4"
},
"outputs": [],
"source": [
"!pip install litellm"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {
"id": "p-MQqWOT_1a7"
},
"outputs": [],
"source": [
"import os\n",
"\n",
"os.environ['OPENROUTER_API_KEY'] = \"\""
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "Ze8JqMqWAARO",
"outputId": "64f3e836-69fa-4f8e-fb35-088a913bbe98"
},
"outputs": [
{
"data": {
"text/plain": [
"<OpenAIObject id=gen-W8FTMSIEorCp3vG5iYIgNMR4IeBv at 0x7c3dcef1f060> JSON: {\n",
" \"id\": \"gen-W8FTMSIEorCp3vG5iYIgNMR4IeBv\",\n",
" \"model\": \"chat-bison@001\",\n",
" \"choices\": [\n",
" {\n",
" \"message\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": \"```\\n#include <stdio.h>\\n\\nint main() {\\n printf(\\\"Hi!\\\\n\\\");\\n return 0;\\n}\\n```\"\n",
" }\n",
" }\n",
" ],\n",
" \"response_ms\": 7817.777999999999\n",
"}"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from litellm import completion\n",
"response = completion(\n",
" model=\"openrouter/google/palm-2-chat-bison\",\n",
" messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
")\n",
"response"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "-LnhELrnAM_J",
"outputId": "d51c7ab7-d761-4bd1-f849-1534d9df4cd0"
},
"outputs": [
{
"data": {
"text/plain": [
"<OpenAIObject id=gen-IiuV7ZNimDufVeutBHrl8ajPuzEh at 0x7c3dcea67560> JSON: {\n",
" \"choices\": [\n",
" {\n",
" \"message\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": \" Here is some simple code to print \\\"Hi\\\":\\n\\n```python\\nprint(\\\"Hi\\\")\\n```\\n\\nThis uses the print() function in Python to output the text \\\"Hi\\\".\"\n",
" },\n",
" \"finish_reason\": \"stop_sequence\"\n",
" }\n",
" ],\n",
" \"model\": \"claude-2.0\",\n",
" \"id\": \"gen-IiuV7ZNimDufVeutBHrl8ajPuzEh\",\n",
" \"response_ms\": 8112.443000000001\n",
"}"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"response = completion(\n",
" model=\"openrouter/anthropic/claude-2\",\n",
" messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
")\n",
"response"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "dJBOUYdwCEn1",
"outputId": "ffa18679-ec15-4dad-fe2b-68665cdf36b0"
},
"outputs": [
{
"data": {
"text/plain": [
"<OpenAIObject id=gen-PyMd3yyJ0aQsCgIY9R8XGZoAtPbl at 0x7c3dceefcae0> JSON: {\n",
" \"id\": \"gen-PyMd3yyJ0aQsCgIY9R8XGZoAtPbl\",\n",
" \"model\": \"togethercomputer/llama-2-70b-chat\",\n",
" \"choices\": [\n",
" {\n",
" \"message\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": \"*gives a sly smile as they type*\\n\\nHey there, handsome. \\ud83d\\ude0f\\n\\nWhat brings you to my neck of the woods today? \\ud83d\\ude18\"\n",
" }\n",
" }\n",
" ],\n",
" \"response_ms\": 9618.775\n",
"}"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"response = completion(\n",
" model=\"openrouter/meta-llama/llama-2-70b-chat\",\n",
" messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
")\n",
"response"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "iFEmsVJI_2BR"
},
"source": [
"# LiteLLM OpenRouter Cookbook"
]
},
"nbformat": 4,
"nbformat_minor": 0
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "cBlUhCEP_xj4"
},
"outputs": [],
"source": [
"!pip install litellm"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {
"id": "p-MQqWOT_1a7"
},
"outputs": [],
"source": [
"import os\n",
"\n",
"os.environ['OPENROUTER_API_KEY'] = \"\""
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "Ze8JqMqWAARO",
"outputId": "64f3e836-69fa-4f8e-fb35-088a913bbe98"
},
"outputs": [
{
"data": {
"text/plain": [
"<OpenAIObject id=gen-W8FTMSIEorCp3vG5iYIgNMR4IeBv at 0x7c3dcef1f060> JSON: {\n",
" \"id\": \"gen-W8FTMSIEorCp3vG5iYIgNMR4IeBv\",\n",
" \"model\": \"chat-bison@001\",\n",
" \"choices\": [\n",
" {\n",
" \"message\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": \"```\\n#include <stdio.h>\\n\\nint main() {\\n printf(\\\"Hi!\\\\n\\\");\\n return 0;\\n}\\n```\"\n",
" }\n",
" }\n",
" ],\n",
" \"response_ms\": 7817.777999999999\n",
"}"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from litellm import completion\n",
"\n",
"response = completion(\n",
" model=\"openrouter/google/palm-2-chat-bison\",\n",
" messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
")\n",
"response"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "-LnhELrnAM_J",
"outputId": "d51c7ab7-d761-4bd1-f849-1534d9df4cd0"
},
"outputs": [
{
"data": {
"text/plain": [
"<OpenAIObject id=gen-IiuV7ZNimDufVeutBHrl8ajPuzEh at 0x7c3dcea67560> JSON: {\n",
" \"choices\": [\n",
" {\n",
" \"message\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": \" Here is some simple code to print \\\"Hi\\\":\\n\\n```python\\nprint(\\\"Hi\\\")\\n```\\n\\nThis uses the print() function in Python to output the text \\\"Hi\\\".\"\n",
" },\n",
" \"finish_reason\": \"stop_sequence\"\n",
" }\n",
" ],\n",
" \"model\": \"claude-2.0\",\n",
" \"id\": \"gen-IiuV7ZNimDufVeutBHrl8ajPuzEh\",\n",
" \"response_ms\": 8112.443000000001\n",
"}"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"response = completion(\n",
" model=\"openrouter/anthropic/claude-2\",\n",
" messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
")\n",
"response"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "dJBOUYdwCEn1",
"outputId": "ffa18679-ec15-4dad-fe2b-68665cdf36b0"
},
"outputs": [
{
"data": {
"text/plain": [
"<OpenAIObject id=gen-PyMd3yyJ0aQsCgIY9R8XGZoAtPbl at 0x7c3dceefcae0> JSON: {\n",
" \"id\": \"gen-PyMd3yyJ0aQsCgIY9R8XGZoAtPbl\",\n",
" \"model\": \"togethercomputer/llama-2-70b-chat\",\n",
" \"choices\": [\n",
" {\n",
" \"message\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": \"*gives a sly smile as they type*\\n\\nHey there, handsome. \\ud83d\\ude0f\\n\\nWhat brings you to my neck of the woods today? \\ud83d\\ude18\"\n",
" }\n",
" }\n",
" ],\n",
" \"response_ms\": 9618.775\n",
"}"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"response = completion(\n",
" model=\"openrouter/meta-llama/llama-2-70b-chat\",\n",
" messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
")\n",
"response"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

256
cookbook/LiteLLM_PromptLayer.ipynb vendored
View file

File diff suppressed because one or more lines are too long

443
cookbook/LiteLLM_User_Based_Rate_Limits.ipynb vendored
View file

@ -1,224 +1,225 @@
{
"cells": [
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "JRCXfhACct4Y"
},
"source": [
"## User Based Rate Limiting Using LiteLLM\n",
"- LiteLLM allows you to set budgets per user\n",
"- Check if a given user has cross their allocated budget\n",
"\n",
"In this notebook we create a $0.0002 daily budget per user and make completion calls using the litellm budget manager"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "fl1kcLG8aaIV"
},
"outputs": [],
"source": [
"!pip install litellm uuid"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "zqRrpoQ3c6oQ"
},
"source": [
"## Imports & Env variables"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"id": "CSkz8bmwdD3w"
},
"outputs": [],
"source": [
"import uuid\n",
"import os\n",
"os.environ['OPENAI_API_KEY'] = \"\""
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "ktqe3gSmdFQ4"
},
"source": [
"## completion() with the budget manager\n",
"\n",
"This code does the following\n",
"- Initializes a litellm.BudgetManager()\n",
"- Checks if a budget exists for a user\n",
" - Creates a $0.0002 budget if the user does not exisr\n",
"- Makes a `litellm.completion()` request only if the user is under their budget"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "pUN48YvmaiRU",
"outputId": "082d6a8b-9aef-4794-9eac-7ba9823ea373"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"No budget exists for user: 29af95f8-c3c6-4c8c-b080-8b2d18d25432\n",
"\n",
"Creating a budget for user: 29af95f8-c3c6-4c8c-b080-8b2d18d25432, daily budget $0.0002\n",
"\n",
"User: 29af95f8-c3c6-4c8c-b080-8b2d18d25432 has spent $0, budget for user: $0.0002\n",
"\n",
"{\n",
" \"id\": \"chatcmpl-7yAUkHQV8xdfldzzZnnnuVU8pl31b\",\n",
" \"object\": \"chat.completion\",\n",
" \"created\": 1694574378,\n",
" \"model\": \"gpt-3.5-turbo-0613\",\n",
" \"choices\": [\n",
" {\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": \"Hello! I'm an AI, so I don't have emotions, but I'm here to assist you. How can I help you today?\"\n",
" },\n",
" \"finish_reason\": \"stop\"\n",
" }\n",
" ],\n",
" \"usage\": {\n",
" \"prompt_tokens\": 14,\n",
" \"completion_tokens\": 29,\n",
" \"total_tokens\": 43\n",
" }\n",
"}\n"
]
},
{
"data": {
"text/plain": [
"{'status': 'success'}"
]
},
"execution_count": 23,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from litellm import BudgetManager, completion\n",
"\n",
"# Initializes a litellm.BudgetManager()\n",
"budget_manager = BudgetManager(project_name=\"liteLLM_project\", client_type=\"hosted\") # see https://docs.litellm.ai/docs/budget_manager\n",
"\n",
"user_id = str(uuid.uuid4()) # create a new user id\n",
"daily_budget = 0.0002\n",
"\n",
"# Checks if a budget exists for a user\n",
"if not budget_manager.is_valid_user(user_id):\n",
" # Creates a $0.0002 budget if the user does not exisr\n",
" print(f\"No budget exists for user: {user_id}\\n\")\n",
" print(f\"Creating a budget for user: {user_id}, daily budget ${daily_budget}\\n\")\n",
" budget_manager.create_budget(total_budget=daily_budget, user=user_id, duration=\"daily\") # duration can be daily, weekly, monthly\n",
"\n",
"\n",
"# Makes a `litellm.completion()` request only if the user is under their budget\n",
"current_spend_for_user = budget_manager.get_current_cost(user=user_id)\n",
"budget_for_user = budget_manager.get_total_budget(user_id)\n",
"print(f\"User: {user_id} has spent ${current_spend_for_user}, budget for user: ${budget_for_user}\\n\")\n",
"\n",
"if current_spend_for_user <= budget_for_user:\n",
" response = completion(model=\"gpt-3.5-turbo\", messages=[{\"role\": \"user\", \"content\": \"Hey, how's it going?\"}])\n",
" budget_manager.update_cost(completion_obj=response, user=user_id)\n",
"else:\n",
" response = \"Sorry - no budget!\"\n",
"\n",
"print(response)"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "yMOirNoBfmmc"
},
"source": [
"## Make 10 calls to cross the budget per user\n",
"- Code fails after user crossed their budget"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "apKF3H-xbFXc",
"outputId": "1c6ef0fe-e27e-4ead-adc6-2c7eb0214e44"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"User: 29af95f8-c3c6-4c8c-b080-8b2d18d25432 has spent $7.9e-05, budget for user: $0.0002\n",
"\n",
"User: 29af95f8-c3c6-4c8c-b080-8b2d18d25432 has spent $0.00015999999999999999, budget for user: $0.0002\n",
"\n",
"User: 29af95f8-c3c6-4c8c-b080-8b2d18d25432 has spent $0.00023899999999999998, budget for user: $0.0002\n",
"\n",
"User: 29af95f8-c3c6-4c8c-b080-8b2d18d25432 has exceeded budget, current spend $0.00023899999999999998, budget for user: $0.0002\n",
"\n"
]
}
],
"source": [
"user_id = \"29af95f8-c3c6-4c8c-b080-8b2d18d25432\" # set in the previous cell\n",
"\n",
"for _ in range(10):\n",
" # check if a given call can be made\n",
" current_spend_for_user = budget_manager.get_current_cost(user=user_id)\n",
" budget_for_user = budget_manager.get_total_budget(user_id)\n",
" print(f\"User: {user_id} has spent ${current_spend_for_user}, budget for user: ${budget_for_user}\\n\")\n",
" if current_spend_for_user <= budget_for_user:\n",
" response = completion(model=\"gpt-3.5-turbo\", messages=[{\"role\": \"user\", \"content\": \"Hey, how's it going?\"}])\n",
" budget_manager.update_cost(completion_obj=response, user=user_id)\n",
" else:\n",
" response = \"Sorry - no budget!\"\n",
" print(f\"User: {user_id} has exceeded budget, current spend ${current_spend_for_user}, budget for user: ${budget_for_user}\\n\")\n",
" break # no more requests\n",
"\n",
" # print(response)"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
"cells": [
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "JRCXfhACct4Y"
},
"source": [
"## User Based Rate Limiting Using LiteLLM\n",
"- LiteLLM allows you to set budgets per user\n",
"- Check if a given user has cross their allocated budget\n",
"\n",
"In this notebook we create a $0.0002 daily budget per user and make completion calls using the litellm budget manager"
]
},
"nbformat": 4,
"nbformat_minor": 0
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "fl1kcLG8aaIV"
},
"outputs": [],
"source": [
"!pip install litellm uuid"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "zqRrpoQ3c6oQ"
},
"source": [
"## Imports & Env variables"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"id": "CSkz8bmwdD3w"
},
"outputs": [],
"source": [
"import os\n",
"import uuid\n",
"\n",
"os.environ['OPENAI_API_KEY'] = \"\""
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "ktqe3gSmdFQ4"
},
"source": [
"## completion() with the budget manager\n",
"\n",
"This code does the following\n",
"- Initializes a litellm.BudgetManager()\n",
"- Checks if a budget exists for a user\n",
" - Creates a $0.0002 budget if the user does not exisr\n",
"- Makes a `litellm.completion()` request only if the user is under their budget"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "pUN48YvmaiRU",
"outputId": "082d6a8b-9aef-4794-9eac-7ba9823ea373"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"No budget exists for user: 29af95f8-c3c6-4c8c-b080-8b2d18d25432\n",
"\n",
"Creating a budget for user: 29af95f8-c3c6-4c8c-b080-8b2d18d25432, daily budget $0.0002\n",
"\n",
"User: 29af95f8-c3c6-4c8c-b080-8b2d18d25432 has spent $0, budget for user: $0.0002\n",
"\n",
"{\n",
" \"id\": \"chatcmpl-7yAUkHQV8xdfldzzZnnnuVU8pl31b\",\n",
" \"object\": \"chat.completion\",\n",
" \"created\": 1694574378,\n",
" \"model\": \"gpt-3.5-turbo-0613\",\n",
" \"choices\": [\n",
" {\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": \"Hello! I'm an AI, so I don't have emotions, but I'm here to assist you. How can I help you today?\"\n",
" },\n",
" \"finish_reason\": \"stop\"\n",
" }\n",
" ],\n",
" \"usage\": {\n",
" \"prompt_tokens\": 14,\n",
" \"completion_tokens\": 29,\n",
" \"total_tokens\": 43\n",
" }\n",
"}\n"
]
},
{
"data": {
"text/plain": [
"{'status': 'success'}"
]
},
"execution_count": 23,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from litellm import BudgetManager, completion\n",
"\n",
"# Initializes a litellm.BudgetManager()\n",
"budget_manager = BudgetManager(project_name=\"liteLLM_project\", client_type=\"hosted\") # see https://docs.litellm.ai/docs/budget_manager\n",
"\n",
"user_id = str(uuid.uuid4()) # create a new user id\n",
"daily_budget = 0.0002\n",
"\n",
"# Checks if a budget exists for a user\n",
"if not budget_manager.is_valid_user(user_id):\n",
" # Creates a $0.0002 budget if the user does not exisr\n",
" print(f\"No budget exists for user: {user_id}\\n\")\n",
" print(f\"Creating a budget for user: {user_id}, daily budget ${daily_budget}\\n\")\n",
" budget_manager.create_budget(total_budget=daily_budget, user=user_id, duration=\"daily\") # duration can be daily, weekly, monthly\n",
"\n",
"\n",
"# Makes a `litellm.completion()` request only if the user is under their budget\n",
"current_spend_for_user = budget_manager.get_current_cost(user=user_id)\n",
"budget_for_user = budget_manager.get_total_budget(user_id)\n",
"print(f\"User: {user_id} has spent ${current_spend_for_user}, budget for user: ${budget_for_user}\\n\")\n",
"\n",
"if current_spend_for_user <= budget_for_user:\n",
" response = completion(model=\"gpt-3.5-turbo\", messages=[{\"role\": \"user\", \"content\": \"Hey, how's it going?\"}])\n",
" budget_manager.update_cost(completion_obj=response, user=user_id)\n",
"else:\n",
" response = \"Sorry - no budget!\"\n",
"\n",
"print(response)"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "yMOirNoBfmmc"
},
"source": [
"## Make 10 calls to cross the budget per user\n",
"- Code fails after user crossed their budget"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "apKF3H-xbFXc",
"outputId": "1c6ef0fe-e27e-4ead-adc6-2c7eb0214e44"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"User: 29af95f8-c3c6-4c8c-b080-8b2d18d25432 has spent $7.9e-05, budget for user: $0.0002\n",
"\n",
"User: 29af95f8-c3c6-4c8c-b080-8b2d18d25432 has spent $0.00015999999999999999, budget for user: $0.0002\n",
"\n",
"User: 29af95f8-c3c6-4c8c-b080-8b2d18d25432 has spent $0.00023899999999999998, budget for user: $0.0002\n",
"\n",
"User: 29af95f8-c3c6-4c8c-b080-8b2d18d25432 has exceeded budget, current spend $0.00023899999999999998, budget for user: $0.0002\n",
"\n"
]
}
],
"source": [
"user_id = \"29af95f8-c3c6-4c8c-b080-8b2d18d25432\" # set in the previous cell\n",
"\n",
"for _ in range(10):\n",
" # check if a given call can be made\n",
" current_spend_for_user = budget_manager.get_current_cost(user=user_id)\n",
" budget_for_user = budget_manager.get_total_budget(user_id)\n",
" print(f\"User: {user_id} has spent ${current_spend_for_user}, budget for user: ${budget_for_user}\\n\")\n",
" if current_spend_for_user <= budget_for_user:\n",
" response = completion(model=\"gpt-3.5-turbo\", messages=[{\"role\": \"user\", \"content\": \"Hey, how's it going?\"}])\n",
" budget_manager.update_cost(completion_obj=response, user=user_id)\n",
" else:\n",
" response = \"Sorry - no budget!\"\n",
" print(f\"User: {user_id} has exceeded budget, current spend ${current_spend_for_user}, budget for user: ${budget_for_user}\\n\")\n",
" break # no more requests\n",
"\n",
" # print(response)"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

1
cookbook/LiteLLM_batch_completion.ipynb vendored
View file

@ -41,6 +41,7 @@
"outputs": [],
"source": [
"import os\n",
"\n",
"from litellm import batch_completion\n",
"\n",
"# set your API_KEY\n",

952
cookbook/Parallel_function_calling.ipynb vendored
View file

@ -1,478 +1,486 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "gHwFJ-srdnku"
},
"source": [
"This is a tutorial on using Parallel function calling with LiteLLM"
]
},
"cells": [
{
"cell_type": "markdown",
"source": [
"This is a tutorial on using Parallel function calling with LiteLLM"
],
"metadata": {
"id": "gHwFJ-srdnku"
}
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "RrtHuVHlZmUe"
},
"outputs": [],
"source": [
"!pip install litellm"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "sG5ANaazjU0g"
},
"source": [
"This tutorial walks through the steps doing parallel function calling using\n",
" - OpenAI\n",
" - Azure OpenAI"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"id": "l4GQ-M5yZ5UW"
},
"outputs": [],
"source": [
"# set openai api key\n",
"import os\n",
"\n",
"os.environ['OPENAI_API_KEY'] = \"\" # litellm reads OPENAI_API_KEY from .env and sends the request"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "AxgR2fCgaRoW"
},
"source": [
"\n",
"# OpenAI gpt-3.5-turbo-1106\n",
"## Step 1: send the conversation and available functions to the model"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "Y3qteFo8ZrZP",
"outputId": "ee6c1183-55c1-4111-cdc0-967b8fed9db3"
},
"outputs": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "RrtHuVHlZmUe"
},
"outputs": [],
"source": [
"!pip install litellm"
]
},
{
"cell_type": "markdown",
"source": [
"This tutorial walks through the steps doing parallel function calling using\n",
" - OpenAI\n",
" - Azure OpenAI"
],
"metadata": {
"id": "sG5ANaazjU0g"
}
},
{
"cell_type": "code",
"source": [
"# set openai api key\n",
"import os\n",
"os.environ['OPENAI_API_KEY'] = \"\" # litellm reads OPENAI_API_KEY from .env and sends the request"
],
"metadata": {
"id": "l4GQ-M5yZ5UW"
},
"execution_count": 3,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"\n",
"# OpenAI gpt-3.5-turbo-1106\n",
"## Step 1: send the conversation and available functions to the model"
],
"metadata": {
"id": "AxgR2fCgaRoW"
}
},
{
"cell_type": "code",
"source": [
"import litellm\n",
"import json\n",
"# Example dummy function hard coded to return the same weather\n",
"# In production, this could be your backend API or an external API\n",
"def get_current_weather(location, unit=\"fahrenheit\"):\n",
" \"\"\"Get the current weather in a given location\"\"\"\n",
" if \"tokyo\" in location.lower():\n",
" return json.dumps({\"location\": \"Tokyo\", \"temperature\": \"10\", \"unit\": \"celsius\"})\n",
" elif \"san francisco\" in location.lower():\n",
" return json.dumps({\"location\": \"San Francisco\", \"temperature\": \"72\", \"unit\": \"fahrenheit\"})\n",
" elif \"paris\" in location.lower():\n",
" return json.dumps({\"location\": \"Paris\", \"temperature\": \"22\", \"unit\": \"celsius\"})\n",
" else:\n",
" return json.dumps({\"location\": location, \"temperature\": \"unknown\"})\n",
"\n",
"messages = [{\"role\": \"user\", \"content\": \"What's the weather like in San Francisco, Tokyo, and Paris?\"}]\n",
"tools = [\n",
" {\n",
" \"type\": \"function\",\n",
" \"function\": {\n",
" \"name\": \"get_current_weather\",\n",
" \"description\": \"Get the current weather in a given location\",\n",
" \"parameters\": {\n",
" \"type\": \"object\",\n",
" \"properties\": {\n",
" \"location\": {\n",
" \"type\": \"string\",\n",
" \"description\": \"The city and state, e.g. San Francisco, CA\",\n",
" },\n",
" \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"]},\n",
" },\n",
" \"required\": [\"location\"],\n",
" },\n",
" },\n",
" }\n",
"]\n",
"\n",
"response = litellm.completion(\n",
" model=\"gpt-3.5-turbo-1106\",\n",
" messages=messages,\n",
" tools=tools,\n",
" tool_choice=\"auto\", # auto is default, but we'll be explicit\n",
")\n",
"print(\"\\nLLM Response1:\\n\", response)\n",
"response_message = response.choices[0].message\n",
"tool_calls = response.choices[0].message.tool_calls\n",
"print(\"\\nTool Choice:\\n\", tool_calls)\n"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "Y3qteFo8ZrZP",
"outputId": "ee6c1183-55c1-4111-cdc0-967b8fed9db3"
},
"execution_count": 18,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"\n",
"LLM Response1:\n",
" ModelResponse(id='chatcmpl-8MNdPbrhtnwiPK1x3PEoGwrH144TW', choices=[Choices(finish_reason='tool_calls', index=0, message=Message(content=None, role='assistant', tool_calls=[ChatCompletionMessageToolCall(id='call_K2Giwoq3NloGPfSv25MJVFZG', function=Function(arguments='{\"location\": \"San Francisco\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function'), ChatCompletionMessageToolCall(id='call_6K8bYCZK6qsbMY3n51FzE5Nz', function=Function(arguments='{\"location\": \"Tokyo\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function'), ChatCompletionMessageToolCall(id='call_cKSmUEJGufDwS7TaUHWzp7qx', function=Function(arguments='{\"location\": \"Paris\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function')]))], created=1700344759, model='gpt-3.5-turbo-1106', object='chat.completion', system_fingerprint='fp_eeff13170a', usage={'completion_tokens': 77, 'prompt_tokens': 88, 'total_tokens': 165}, _response_ms=1049.913)\n",
"\n",
"Tool Choice:\n",
" [ChatCompletionMessageToolCall(id='call_K2Giwoq3NloGPfSv25MJVFZG', function=Function(arguments='{\"location\": \"San Francisco\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function'), ChatCompletionMessageToolCall(id='call_6K8bYCZK6qsbMY3n51FzE5Nz', function=Function(arguments='{\"location\": \"Tokyo\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function'), ChatCompletionMessageToolCall(id='call_cKSmUEJGufDwS7TaUHWzp7qx', function=Function(arguments='{\"location\": \"Paris\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function')]\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"## Step 2 - Parse the Model Response and Execute Functions"
],
"metadata": {
"id": "tD4lJQ40cU44"
}
},
{
"cell_type": "code",
"source": [
"# Check if the model wants to call a function\n",
"if tool_calls:\n",
" # Execute the functions and prepare responses\n",
" available_functions = {\n",
" \"get_current_weather\": get_current_weather,\n",
" }\n",
"\n",
" messages.append(response_message) # Extend conversation with assistant's reply\n",
"\n",
" for tool_call in tool_calls:\n",
" print(f\"\\nExecuting tool call\\n{tool_call}\")\n",
" function_name = tool_call.function.name\n",
" function_to_call = available_functions[function_name]\n",
" function_args = json.loads(tool_call.function.arguments)\n",
" # calling the get_current_weather() function\n",
" function_response = function_to_call(\n",
" location=function_args.get(\"location\"),\n",
" unit=function_args.get(\"unit\"),\n",
" )\n",
" print(f\"Result from tool call\\n{function_response}\\n\")\n",
"\n",
" # Extend conversation with function response\n",
" messages.append(\n",
" {\n",
" \"tool_call_id\": tool_call.id,\n",
" \"role\": \"tool\",\n",
" \"name\": function_name,\n",
" \"content\": function_response,\n",
" }\n",
" )\n"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "af4oXQvicV_n",
"outputId": "abf6ac3e-4a21-4a4f-b8d7-809b763d0632"
},
"execution_count": 21,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"\n",
"Executing tool call\n",
"ChatCompletionMessageToolCall(id='call_K2Giwoq3NloGPfSv25MJVFZG', function=Function(arguments='{\"location\": \"San Francisco\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function')\n",
"Result from tool call\n",
"{\"location\": \"San Francisco\", \"temperature\": \"72\", \"unit\": \"fahrenheit\"}\n",
"\n",
"\n",
"Executing tool call\n",
"ChatCompletionMessageToolCall(id='call_6K8bYCZK6qsbMY3n51FzE5Nz', function=Function(arguments='{\"location\": \"Tokyo\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function')\n",
"Result from tool call\n",
"{\"location\": \"Tokyo\", \"temperature\": \"10\", \"unit\": \"celsius\"}\n",
"\n",
"\n",
"Executing tool call\n",
"ChatCompletionMessageToolCall(id='call_cKSmUEJGufDwS7TaUHWzp7qx', function=Function(arguments='{\"location\": \"Paris\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function')\n",
"Result from tool call\n",
"{\"location\": \"Paris\", \"temperature\": \"22\", \"unit\": \"celsius\"}\n",
"\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"## Step 3 - Second litellm.completion() call"
],
"metadata": {
"id": "E3OL1fqUdFdv"
}
},
{
"cell_type": "code",
"source": [
"second_response = litellm.completion(\n",
" model=\"gpt-3.5-turbo-1106\",\n",
" messages=messages,\n",
")\n",
"print(\"Second Response\\n\", second_response)\n",
"print(\"Second Response Message\\n\", second_response.choices[0].message.content)\n"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "8KYB2n-jc1_f",
"outputId": "6c6448ae-1c09-43ae-eb90-208b118e6179"
},
"execution_count": 26,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Second Response\n",
" ModelResponse(id='chatcmpl-8MNhat166ZqjO6egXcUh85Pd0s7KV', choices=[Choices(finish_reason='stop', index=0, message=Message(content=\"The current weather in San Francisco is 72°F, in Tokyo it's 10°C, and in Paris it's 22°C.\", role='assistant'))], created=1700345018, model='gpt-3.5-turbo-1106', object='chat.completion', system_fingerprint='fp_eeff13170a', usage={'completion_tokens': 28, 'prompt_tokens': 465, 'total_tokens': 493}, _response_ms=999.246)\n",
"Second Response Message\n",
" The current weather in San Francisco is 72°F, in Tokyo it's 10°C, and in Paris it's 22°C.\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"## Using Azure OpenAI"
],
"metadata": {
"id": "1cIIFEvXjofp"
}
},
{
"cell_type": "code",
"source": [
"# set Azure env variables\n",
"import os\n",
"os.environ['AZURE_API_KEY'] = \"\" # litellm reads AZURE_API_KEY from .env and sends the request\n",
"os.environ['AZURE_API_BASE'] = \"https://openai-gpt-4-test-v-1.openai.azure.com/\"\n",
"os.environ['AZURE_API_VERSION'] = \"2023-07-01-preview\""
],
"metadata": {
"id": "lG9mUnModeeE"
},
"execution_count": 32,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"## Step 1"
],
"metadata": {
"id": "17S-Ysksj-E_"
}
},
{
"cell_type": "code",
"source": [
"import litellm\n",
"import json\n",
"# Example dummy function hard coded to return the same weather\n",
"# In production, this could be your backend API or an external API\n",
"def get_current_weather(location, unit=\"fahrenheit\"):\n",
" \"\"\"Get the current weather in a given location\"\"\"\n",
" if \"tokyo\" in location.lower():\n",
" return json.dumps({\"location\": \"Tokyo\", \"temperature\": \"10\", \"unit\": \"celsius\"})\n",
" elif \"san francisco\" in location.lower():\n",
" return json.dumps({\"location\": \"San Francisco\", \"temperature\": \"72\", \"unit\": \"fahrenheit\"})\n",
" elif \"paris\" in location.lower():\n",
" return json.dumps({\"location\": \"Paris\", \"temperature\": \"22\", \"unit\": \"celsius\"})\n",
" else:\n",
" return json.dumps({\"location\": location, \"temperature\": \"unknown\"})\n",
"\n",
"messages = [{\"role\": \"user\", \"content\": \"What's the weather like in San Francisco, Tokyo, and Paris?\"}]\n",
"tools = [\n",
" {\n",
" \"type\": \"function\",\n",
" \"function\": {\n",
" \"name\": \"get_current_weather\",\n",
" \"description\": \"Get the current weather in a given location\",\n",
" \"parameters\": {\n",
" \"type\": \"object\",\n",
" \"properties\": {\n",
" \"location\": {\n",
" \"type\": \"string\",\n",
" \"description\": \"The city and state, e.g. San Francisco, CA\",\n",
" },\n",
" \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"]},\n",
" },\n",
" \"required\": [\"location\"],\n",
" },\n",
" },\n",
" }\n",
"]\n",
"\n",
"response = litellm.completion(\n",
" model=\"azure/chatgpt-functioncalling\", # model = azure/<your-azure-deployment-name>\n",
" messages=messages,\n",
" tools=tools,\n",
" tool_choice=\"auto\", # auto is default, but we'll be explicit\n",
")\n",
"print(\"\\nLLM Response1:\\n\", response)\n",
"response_message = response.choices[0].message\n",
"tool_calls = response.choices[0].message.tool_calls\n",
"print(\"\\nTool Choice:\\n\", tool_calls)\n"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "boAIHLEXj80m",
"outputId": "00afcf09-5b6b-4805-c374-ba089cc6eb43"
},
"execution_count": 33,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"\n",
"LLM Response1:\n",
" ModelResponse(id='chatcmpl-8MOBPvEnqG7qitkmVqZmCrzSGEmDj', choices=[Choices(finish_reason='tool_calls', index=0, message=Message(content=None, role='assistant', tool_calls=[ChatCompletionMessageToolCall(id='call_7gZ0PkmmmgzTOxfF01ATp0U5', function=Function(arguments='{\\n \"location\": \"San Francisco, CA\"\\n}', name='get_current_weather'), type='function')]))], created=1700346867, model='gpt-35-turbo', object='chat.completion', system_fingerprint=None, usage={'completion_tokens': 19, 'prompt_tokens': 88, 'total_tokens': 107}, _response_ms=833.4319999999999)\n",
"\n",
"Tool Choice:\n",
" [ChatCompletionMessageToolCall(id='call_7gZ0PkmmmgzTOxfF01ATp0U5', function=Function(arguments='{\\n \"location\": \"San Francisco, CA\"\\n}', name='get_current_weather'), type='function')]\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"## Step 2"
],
"metadata": {
"id": "hqh1y1IMkmGO"
}
},
{
"cell_type": "code",
"source": [
"# Check if the model wants to call a function\n",
"if tool_calls:\n",
" # Execute the functions and prepare responses\n",
" available_functions = {\n",
" \"get_current_weather\": get_current_weather,\n",
" }\n",
"\n",
" messages.append(response_message) # Extend conversation with assistant's reply\n",
"\n",
" for tool_call in tool_calls:\n",
" print(f\"\\nExecuting tool call\\n{tool_call}\")\n",
" function_name = tool_call.function.name\n",
" function_to_call = available_functions[function_name]\n",
" function_args = json.loads(tool_call.function.arguments)\n",
" # calling the get_current_weather() function\n",
" function_response = function_to_call(\n",
" location=function_args.get(\"location\"),\n",
" unit=function_args.get(\"unit\"),\n",
" )\n",
" print(f\"Result from tool call\\n{function_response}\\n\")\n",
"\n",
" # Extend conversation with function response\n",
" messages.append(\n",
" {\n",
" \"tool_call_id\": tool_call.id,\n",
" \"role\": \"tool\",\n",
" \"name\": function_name,\n",
" \"content\": function_response,\n",
" }\n",
" )\n"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "FGu7DY7PkOiG",
"outputId": "96d39ae7-7fc8-4dd8-c82f-5ee9a486724c"
},
"execution_count": 34,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"\n",
"Executing tool call\n",
"ChatCompletionMessageToolCall(id='call_7gZ0PkmmmgzTOxfF01ATp0U5', function=Function(arguments='{\\n \"location\": \"San Francisco, CA\"\\n}', name='get_current_weather'), type='function')\n",
"Result from tool call\n",
"{\"location\": \"San Francisco\", \"temperature\": \"72\", \"unit\": \"fahrenheit\"}\n",
"\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"## Step 3"
],
"metadata": {
"id": "4MjYyeajkpBl"
}
},
{
"cell_type": "code",
"source": [
"second_response = litellm.completion(\n",
" model=\"azure/chatgpt-functioncalling\",\n",
" messages=messages,\n",
")\n",
"print(\"Second Response\\n\", second_response)\n",
"print(\"Second Response Message\\n\", second_response.choices[0].message.content)\n"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "qHgXyZq1kqGn",
"outputId": "61a30470-d7f5-484d-c42b-681c9b60b34a"
},
"execution_count": 36,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Second Response\n",
" ModelResponse(id='chatcmpl-8MOC90vwZ2LHX0DE796XYtsOxdGcc', choices=[Choices(finish_reason='stop', index=0, message=Message(content='The current weather in San Francisco is 72°F.', role='assistant'))], created=1700346913, model='gpt-35-turbo', object='chat.completion', system_fingerprint=None, usage={'completion_tokens': 11, 'prompt_tokens': 69, 'total_tokens': 80}, _response_ms=824.882)\n",
"Second Response Message\n",
" The current weather in San Francisco is 72°F.\n"
]
}
]
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"LLM Response1:\n",
" ModelResponse(id='chatcmpl-8MNdPbrhtnwiPK1x3PEoGwrH144TW', choices=[Choices(finish_reason='tool_calls', index=0, message=Message(content=None, role='assistant', tool_calls=[ChatCompletionMessageToolCall(id='call_K2Giwoq3NloGPfSv25MJVFZG', function=Function(arguments='{\"location\": \"San Francisco\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function'), ChatCompletionMessageToolCall(id='call_6K8bYCZK6qsbMY3n51FzE5Nz', function=Function(arguments='{\"location\": \"Tokyo\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function'), ChatCompletionMessageToolCall(id='call_cKSmUEJGufDwS7TaUHWzp7qx', function=Function(arguments='{\"location\": \"Paris\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function')]))], created=1700344759, model='gpt-3.5-turbo-1106', object='chat.completion', system_fingerprint='fp_eeff13170a', usage={'completion_tokens': 77, 'prompt_tokens': 88, 'total_tokens': 165}, _response_ms=1049.913)\n",
"\n",
"Tool Choice:\n",
" [ChatCompletionMessageToolCall(id='call_K2Giwoq3NloGPfSv25MJVFZG', function=Function(arguments='{\"location\": \"San Francisco\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function'), ChatCompletionMessageToolCall(id='call_6K8bYCZK6qsbMY3n51FzE5Nz', function=Function(arguments='{\"location\": \"Tokyo\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function'), ChatCompletionMessageToolCall(id='call_cKSmUEJGufDwS7TaUHWzp7qx', function=Function(arguments='{\"location\": \"Paris\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function')]\n"
]
}
]
],
"source": [
"import json\n",
"\n",
"import litellm\n",
"\n",
"\n",
"# Example dummy function hard coded to return the same weather\n",
"# In production, this could be your backend API or an external API\n",
"def get_current_weather(location, unit=\"fahrenheit\"):\n",
" \"\"\"Get the current weather in a given location\"\"\"\n",
" if \"tokyo\" in location.lower():\n",
" return json.dumps({\"location\": \"Tokyo\", \"temperature\": \"10\", \"unit\": \"celsius\"})\n",
" elif \"san francisco\" in location.lower():\n",
" return json.dumps({\"location\": \"San Francisco\", \"temperature\": \"72\", \"unit\": \"fahrenheit\"})\n",
" elif \"paris\" in location.lower():\n",
" return json.dumps({\"location\": \"Paris\", \"temperature\": \"22\", \"unit\": \"celsius\"})\n",
" else:\n",
" return json.dumps({\"location\": location, \"temperature\": \"unknown\"})\n",
"\n",
"messages = [{\"role\": \"user\", \"content\": \"What's the weather like in San Francisco, Tokyo, and Paris?\"}]\n",
"tools = [\n",
" {\n",
" \"type\": \"function\",\n",
" \"function\": {\n",
" \"name\": \"get_current_weather\",\n",
" \"description\": \"Get the current weather in a given location\",\n",
" \"parameters\": {\n",
" \"type\": \"object\",\n",
" \"properties\": {\n",
" \"location\": {\n",
" \"type\": \"string\",\n",
" \"description\": \"The city and state, e.g. San Francisco, CA\",\n",
" },\n",
" \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"]},\n",
" },\n",
" \"required\": [\"location\"],\n",
" },\n",
" },\n",
" }\n",
"]\n",
"\n",
"response = litellm.completion(\n",
" model=\"gpt-3.5-turbo-1106\",\n",
" messages=messages,\n",
" tools=tools,\n",
" tool_choice=\"auto\", # auto is default, but we'll be explicit\n",
")\n",
"print(\"\\nLLM Response1:\\n\", response)\n",
"response_message = response.choices[0].message\n",
"tool_calls = response.choices[0].message.tool_calls\n",
"print(\"\\nTool Choice:\\n\", tool_calls)\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "tD4lJQ40cU44"
},
"source": [
"## Step 2 - Parse the Model Response and Execute Functions"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "af4oXQvicV_n",
"outputId": "abf6ac3e-4a21-4a4f-b8d7-809b763d0632"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Executing tool call\n",
"ChatCompletionMessageToolCall(id='call_K2Giwoq3NloGPfSv25MJVFZG', function=Function(arguments='{\"location\": \"San Francisco\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function')\n",
"Result from tool call\n",
"{\"location\": \"San Francisco\", \"temperature\": \"72\", \"unit\": \"fahrenheit\"}\n",
"\n",
"\n",
"Executing tool call\n",
"ChatCompletionMessageToolCall(id='call_6K8bYCZK6qsbMY3n51FzE5Nz', function=Function(arguments='{\"location\": \"Tokyo\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function')\n",
"Result from tool call\n",
"{\"location\": \"Tokyo\", \"temperature\": \"10\", \"unit\": \"celsius\"}\n",
"\n",
"\n",
"Executing tool call\n",
"ChatCompletionMessageToolCall(id='call_cKSmUEJGufDwS7TaUHWzp7qx', function=Function(arguments='{\"location\": \"Paris\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function')\n",
"Result from tool call\n",
"{\"location\": \"Paris\", \"temperature\": \"22\", \"unit\": \"celsius\"}\n",
"\n"
]
}
],
"source": [
"# Check if the model wants to call a function\n",
"if tool_calls:\n",
" # Execute the functions and prepare responses\n",
" available_functions = {\n",
" \"get_current_weather\": get_current_weather,\n",
" }\n",
"\n",
" messages.append(response_message) # Extend conversation with assistant's reply\n",
"\n",
" for tool_call in tool_calls:\n",
" print(f\"\\nExecuting tool call\\n{tool_call}\")\n",
" function_name = tool_call.function.name\n",
" function_to_call = available_functions[function_name]\n",
" function_args = json.loads(tool_call.function.arguments)\n",
" # calling the get_current_weather() function\n",
" function_response = function_to_call(\n",
" location=function_args.get(\"location\"),\n",
" unit=function_args.get(\"unit\"),\n",
" )\n",
" print(f\"Result from tool call\\n{function_response}\\n\")\n",
"\n",
" # Extend conversation with function response\n",
" messages.append(\n",
" {\n",
" \"tool_call_id\": tool_call.id,\n",
" \"role\": \"tool\",\n",
" \"name\": function_name,\n",
" \"content\": function_response,\n",
" }\n",
" )\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "E3OL1fqUdFdv"
},
"source": [
"## Step 3 - Second litellm.completion() call"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "8KYB2n-jc1_f",
"outputId": "6c6448ae-1c09-43ae-eb90-208b118e6179"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Second Response\n",
" ModelResponse(id='chatcmpl-8MNhat166ZqjO6egXcUh85Pd0s7KV', choices=[Choices(finish_reason='stop', index=0, message=Message(content=\"The current weather in San Francisco is 72°F, in Tokyo it's 10°C, and in Paris it's 22°C.\", role='assistant'))], created=1700345018, model='gpt-3.5-turbo-1106', object='chat.completion', system_fingerprint='fp_eeff13170a', usage={'completion_tokens': 28, 'prompt_tokens': 465, 'total_tokens': 493}, _response_ms=999.246)\n",
"Second Response Message\n",
" The current weather in San Francisco is 72°F, in Tokyo it's 10°C, and in Paris it's 22°C.\n"
]
}
],
"source": [
"second_response = litellm.completion(\n",
" model=\"gpt-3.5-turbo-1106\",\n",
" messages=messages,\n",
")\n",
"print(\"Second Response\\n\", second_response)\n",
"print(\"Second Response Message\\n\", second_response.choices[0].message.content)\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "1cIIFEvXjofp"
},
"source": [
"## Using Azure OpenAI"
]
},
{
"cell_type": "code",
"execution_count": 32,
"metadata": {
"id": "lG9mUnModeeE"
},
"outputs": [],
"source": [
"# set Azure env variables\n",
"import os\n",
"\n",
"os.environ['AZURE_API_KEY'] = \"\" # litellm reads AZURE_API_KEY from .env and sends the request\n",
"os.environ['AZURE_API_BASE'] = \"https://openai-gpt-4-test-v-1.openai.azure.com/\"\n",
"os.environ['AZURE_API_VERSION'] = \"2023-07-01-preview\""
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "17S-Ysksj-E_"
},
"source": [
"## Step 1"
]
},
{
"cell_type": "code",
"execution_count": 33,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "boAIHLEXj80m",
"outputId": "00afcf09-5b6b-4805-c374-ba089cc6eb43"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"LLM Response1:\n",
" ModelResponse(id='chatcmpl-8MOBPvEnqG7qitkmVqZmCrzSGEmDj', choices=[Choices(finish_reason='tool_calls', index=0, message=Message(content=None, role='assistant', tool_calls=[ChatCompletionMessageToolCall(id='call_7gZ0PkmmmgzTOxfF01ATp0U5', function=Function(arguments='{\\n \"location\": \"San Francisco, CA\"\\n}', name='get_current_weather'), type='function')]))], created=1700346867, model='gpt-35-turbo', object='chat.completion', system_fingerprint=None, usage={'completion_tokens': 19, 'prompt_tokens': 88, 'total_tokens': 107}, _response_ms=833.4319999999999)\n",
"\n",
"Tool Choice:\n",
" [ChatCompletionMessageToolCall(id='call_7gZ0PkmmmgzTOxfF01ATp0U5', function=Function(arguments='{\\n \"location\": \"San Francisco, CA\"\\n}', name='get_current_weather'), type='function')]\n"
]
}
],
"source": [
"import json\n",
"\n",
"import litellm\n",
"\n",
"\n",
"# Example dummy function hard coded to return the same weather\n",
"# In production, this could be your backend API or an external API\n",
"def get_current_weather(location, unit=\"fahrenheit\"):\n",
" \"\"\"Get the current weather in a given location\"\"\"\n",
" if \"tokyo\" in location.lower():\n",
" return json.dumps({\"location\": \"Tokyo\", \"temperature\": \"10\", \"unit\": \"celsius\"})\n",
" elif \"san francisco\" in location.lower():\n",
" return json.dumps({\"location\": \"San Francisco\", \"temperature\": \"72\", \"unit\": \"fahrenheit\"})\n",
" elif \"paris\" in location.lower():\n",
" return json.dumps({\"location\": \"Paris\", \"temperature\": \"22\", \"unit\": \"celsius\"})\n",
" else:\n",
" return json.dumps({\"location\": location, \"temperature\": \"unknown\"})\n",
"\n",
"messages = [{\"role\": \"user\", \"content\": \"What's the weather like in San Francisco, Tokyo, and Paris?\"}]\n",
"tools = [\n",
" {\n",
" \"type\": \"function\",\n",
" \"function\": {\n",
" \"name\": \"get_current_weather\",\n",
" \"description\": \"Get the current weather in a given location\",\n",
" \"parameters\": {\n",
" \"type\": \"object\",\n",
" \"properties\": {\n",
" \"location\": {\n",
" \"type\": \"string\",\n",
" \"description\": \"The city and state, e.g. San Francisco, CA\",\n",
" },\n",
" \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"]},\n",
" },\n",
" \"required\": [\"location\"],\n",
" },\n",
" },\n",
" }\n",
"]\n",
"\n",
"response = litellm.completion(\n",
" model=\"azure/chatgpt-functioncalling\", # model = azure/<your-azure-deployment-name>\n",
" messages=messages,\n",
" tools=tools,\n",
" tool_choice=\"auto\", # auto is default, but we'll be explicit\n",
")\n",
"print(\"\\nLLM Response1:\\n\", response)\n",
"response_message = response.choices[0].message\n",
"tool_calls = response.choices[0].message.tool_calls\n",
"print(\"\\nTool Choice:\\n\", tool_calls)\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "hqh1y1IMkmGO"
},
"source": [
"## Step 2"
]
},
{
"cell_type": "code",
"execution_count": 34,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "FGu7DY7PkOiG",
"outputId": "96d39ae7-7fc8-4dd8-c82f-5ee9a486724c"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Executing tool call\n",
"ChatCompletionMessageToolCall(id='call_7gZ0PkmmmgzTOxfF01ATp0U5', function=Function(arguments='{\\n \"location\": \"San Francisco, CA\"\\n}', name='get_current_weather'), type='function')\n",
"Result from tool call\n",
"{\"location\": \"San Francisco\", \"temperature\": \"72\", \"unit\": \"fahrenheit\"}\n",
"\n"
]
}
],
"source": [
"# Check if the model wants to call a function\n",
"if tool_calls:\n",
" # Execute the functions and prepare responses\n",
" available_functions = {\n",
" \"get_current_weather\": get_current_weather,\n",
" }\n",
"\n",
" messages.append(response_message) # Extend conversation with assistant's reply\n",
"\n",
" for tool_call in tool_calls:\n",
" print(f\"\\nExecuting tool call\\n{tool_call}\")\n",
" function_name = tool_call.function.name\n",
" function_to_call = available_functions[function_name]\n",
" function_args = json.loads(tool_call.function.arguments)\n",
" # calling the get_current_weather() function\n",
" function_response = function_to_call(\n",
" location=function_args.get(\"location\"),\n",
" unit=function_args.get(\"unit\"),\n",
" )\n",
" print(f\"Result from tool call\\n{function_response}\\n\")\n",
"\n",
" # Extend conversation with function response\n",
" messages.append(\n",
" {\n",
" \"tool_call_id\": tool_call.id,\n",
" \"role\": \"tool\",\n",
" \"name\": function_name,\n",
" \"content\": function_response,\n",
" }\n",
" )\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "4MjYyeajkpBl"
},
"source": [
"## Step 3"
]
},
{
"cell_type": "code",
"execution_count": 36,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "qHgXyZq1kqGn",
"outputId": "61a30470-d7f5-484d-c42b-681c9b60b34a"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Second Response\n",
" ModelResponse(id='chatcmpl-8MOC90vwZ2LHX0DE796XYtsOxdGcc', choices=[Choices(finish_reason='stop', index=0, message=Message(content='The current weather in San Francisco is 72°F.', role='assistant'))], created=1700346913, model='gpt-35-turbo', object='chat.completion', system_fingerprint=None, usage={'completion_tokens': 11, 'prompt_tokens': 69, 'total_tokens': 80}, _response_ms=824.882)\n",
"Second Response Message\n",
" The current weather in San Francisco is 72°F.\n"
]
}
],
"source": [
"second_response = litellm.completion(\n",
" model=\"azure/chatgpt-functioncalling\",\n",
" messages=messages,\n",
")\n",
"print(\"Second Response\\n\", second_response)\n",
"print(\"Second Response Message\\n\", second_response.choices[0].message.content)\n"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

9
cookbook/Proxy_Batch_Users.ipynb vendored
View file

@ -17,11 +17,12 @@
},
"outputs": [],
"source": [
"import csv\n",
"from typing import Optional\n",
"import httpx\n",
"import json\n",
"import asyncio\n",
"import csv\n",
"import json\n",
"from typing import Optional\n",
"\n",
"import httpx\n",
"\n",
"proxy_base_url = \"http://0.0.0.0:4000\" # 👈 SET TO PROXY URL\n",
"master_key = \"sk-1234\" # 👈 SET TO PROXY MASTER KEY"

2
cookbook/TogetherAI_liteLLM.ipynb vendored
View file

@ -70,7 +70,9 @@
"outputs": [],
"source": [
"import os\n",
"\n",
"from litellm import completion\n",
"\n",
"os.environ[\"TOGETHERAI_API_KEY\"] = \"\" #@param\n",
"user_message = \"Hello, whats the weather in San Francisco??\"\n",
"messages = [{ \"content\": user_message,\"role\": \"user\"}]"

797
cookbook/VLLM_Model_Testing.ipynb vendored
View file

@ -1,404 +1,405 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": [],
"machine_shape": "hm",
"gpuType": "V100"
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
},
"accelerator": "GPU"
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "vDOm5wfjdFLP"
},
"source": [
"# Set up Environment"
]
},
"cells": [
{
"cell_type": "markdown",
"source": [
"# Set up Environment"
],
"metadata": {
"id": "vDOm5wfjdFLP"
}
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "Bx6mAA6MHiy_"
},
"outputs": [],
"source": [
"!pip install --upgrade litellm"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "zIYv7JTyxSxR",
"outputId": "53890320-f9fa-4bf4-8362-0f17f52c6ed4"
},
"outputs": [
{
"cell_type": "code",
"source": [
"!pip install --upgrade litellm"
],
"metadata": {
"id": "Bx6mAA6MHiy_"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "zIYv7JTyxSxR",
"outputId": "53890320-f9fa-4bf4-8362-0f17f52c6ed4"
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Successfully installed fastapi-0.103.1 h11-0.14.0 huggingface-hub-0.16.4 ninja-1.11.1 pydantic-1.10.12 ray-2.6.3 safetensors-0.3.3 sentencepiece-0.1.99 starlette-0.27.0 tokenizers-0.13.3 transformers-4.33.1 uvicorn-0.23.2 vllm-0.1.4 xformers-0.0.21\n"
]
}
],
"source": [
"!pip install vllm"
]
},
{
"cell_type": "markdown",
"source": [
"# Load the Logs"
],
"metadata": {
"id": "RMcoAni6WKEx"
}
},
{
"cell_type": "code",
"source": [
"import pandas as pd"
],
"metadata": {
"id": "zchxB8c7WJe5"
},
"execution_count": 4,
"outputs": []
},
{
"cell_type": "code",
"source": [
"# path of the csv file\n",
"file_path = 'Model-prompts-example.csv'\n",
"\n",
"# load the csv file as a pandas DataFrame\n",
"data = pd.read_csv(file_path)\n",
"\n",
"data.head()"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 81
},
"id": "aKcWr015WNPm",
"outputId": "6e226773-333f-46a2-9fc8-4f54f309d204"
},
"execution_count": 6,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
" Success Timestamp Input \\\n",
"0 True 1694041195 This is the templated query input \n",
"\n",
" Output RunId (Wandb Runid) \\\n",
"0 This is the query output from the model 8hlumwuk \n",
"\n",
" Model ID (or Name) \n",
"0 OpenAI/Turbo-3.5 "
],
"text/html": [
"\n",
" <div id=\"df-cd06d09e-fb43-41b0-938f-37f9d285ae66\" class=\"colab-df-container\">\n",
" <div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Success</th>\n",
" <th>Timestamp</th>\n",
" <th>Input</th>\n",
" <th>Output</th>\n",
" <th>RunId (Wandb Runid)</th>\n",
" <th>Model ID (or Name)</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>True</td>\n",
" <td>1694041195</td>\n",
" <td>This is the templated query input</td>\n",
" <td>This is the query output from the model</td>\n",
" <td>8hlumwuk</td>\n",
" <td>OpenAI/Turbo-3.5</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>\n",
" <div class=\"colab-df-buttons\">\n",
"\n",
" <div class=\"colab-df-container\">\n",
" <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-cd06d09e-fb43-41b0-938f-37f9d285ae66')\"\n",
" title=\"Convert this dataframe to an interactive table.\"\n",
" style=\"display:none;\">\n",
"\n",
" <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
" <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
" </svg>\n",
" </button>\n",
"\n",
" <style>\n",
" .colab-df-container {\n",
" display:flex;\n",
" gap: 12px;\n",
" }\n",
"\n",
" .colab-df-convert {\n",
" background-color: #E8F0FE;\n",
" border: none;\n",
" border-radius: 50%;\n",
" cursor: pointer;\n",
" display: none;\n",
" fill: #1967D2;\n",
" height: 32px;\n",
" padding: 0 0 0 0;\n",
" width: 32px;\n",
" }\n",
"\n",
" .colab-df-convert:hover {\n",
" background-color: #E2EBFA;\n",
" box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
" fill: #174EA6;\n",
" }\n",
"\n",
" .colab-df-buttons div {\n",
" margin-bottom: 4px;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert {\n",
" background-color: #3B4455;\n",
" fill: #D2E3FC;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert:hover {\n",
" background-color: #434B5C;\n",
" box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
" filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
" fill: #FFFFFF;\n",
" }\n",
" </style>\n",
"\n",
" <script>\n",
" const buttonEl =\n",
" document.querySelector('#df-cd06d09e-fb43-41b0-938f-37f9d285ae66 button.colab-df-convert');\n",
" buttonEl.style.display =\n",
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
"\n",
" async function convertToInteractive(key) {\n",
" const element = document.querySelector('#df-cd06d09e-fb43-41b0-938f-37f9d285ae66');\n",
" const dataTable =\n",
" await google.colab.kernel.invokeFunction('convertToInteractive',\n",
" [key], {});\n",
" if (!dataTable) return;\n",
"\n",
" const docLinkHtml = 'Like what you see? Visit the ' +\n",
" '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
" + ' to learn more about interactive tables.';\n",
" element.innerHTML = '';\n",
" dataTable['output_type'] = 'display_data';\n",
" await google.colab.output.renderOutput(dataTable, element);\n",
" const docLink = document.createElement('div');\n",
" docLink.innerHTML = docLinkHtml;\n",
" element.appendChild(docLink);\n",
" }\n",
" </script>\n",
" </div>\n",
"\n",
"\n",
" </div>\n",
" </div>\n"
]
},
"metadata": {},
"execution_count": 6
}
]
},
{
"cell_type": "code",
"source": [
"input_texts = data['Input'].values"
],
"metadata": {
"id": "0DbL-kirWUyn"
},
"execution_count": 7,
"outputs": []
},
{
"cell_type": "code",
"source": [
"messages = [[{\"role\": \"user\", \"content\": input_text}] for input_text in input_texts]"
],
"metadata": {
"id": "cqpAvy8hWXyC"
},
"execution_count": 8,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"# Running Inference"
],
"metadata": {
"id": "SugCyom0Xy8U"
}
},
{
"cell_type": "code",
"source": [
"from litellm import batch_completion\n",
"model_name = \"facebook/opt-125m\"\n",
"provider = \"vllm\"\n",
"response_list = batch_completion(\n",
" model=model_name,\n",
" custom_llm_provider=provider, # can easily switch to huggingface, replicate, together ai, sagemaker, etc.\n",
" messages=messages,\n",
" temperature=0.2,\n",
" max_tokens=80,\n",
" )"
],
"metadata": {
"id": "qpikx3uxHns3"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"response_list"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "QDPikHtwKJJ2",
"outputId": "06f47c44-e258-452a-f9db-232a5b6d2810"
},
"execution_count": 10,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"[<ModelResponse at 0x7e5b87616750> JSON: {\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \".\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is\",\n",
" \"role\": \"assistant\",\n",
" \"logprobs\": null\n",
" }\n",
" }\n",
" ],\n",
" \"created\": 1694053363.6139505,\n",
" \"model\": \"facebook/opt-125m\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": 9,\n",
" \"completion_tokens\": 80,\n",
" \"total_tokens\": 89\n",
" }\n",
" }]"
]
},
"metadata": {},
"execution_count": 10
}
]
},
{
"cell_type": "code",
"source": [
"response_values = [response['choices'][0]['message']['content'] for response in response_list]"
],
"metadata": {
"id": "SYqTcCiJbQDF"
},
"execution_count": 11,
"outputs": []
},
{
"cell_type": "code",
"source": [
"response_values"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "wqs-Oy9FbiPo",
"outputId": "16a6a7b7-97c8-4b5b-eff8-09ea5eb5ad06"
},
"execution_count": 12,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"['.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is']"
]
},
"metadata": {},
"execution_count": 12
}
]
},
{
"cell_type": "code",
"source": [
"data[f\"{model_name}_output\"] = response_values"
],
"metadata": {
"id": "mElNbBehbkrz"
},
"execution_count": 13,
"outputs": []
},
{
"cell_type": "code",
"source": [
"data.to_csv('model_responses.csv', index=False)"
],
"metadata": {
"id": "F06NXssDc45k"
},
"execution_count": 14,
"outputs": []
"name": "stdout",
"output_type": "stream",
"text": [
"Successfully installed fastapi-0.103.1 h11-0.14.0 huggingface-hub-0.16.4 ninja-1.11.1 pydantic-1.10.12 ray-2.6.3 safetensors-0.3.3 sentencepiece-0.1.99 starlette-0.27.0 tokenizers-0.13.3 transformers-4.33.1 uvicorn-0.23.2 vllm-0.1.4 xformers-0.0.21\n"
]
}
]
],
"source": [
"!pip install vllm"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "RMcoAni6WKEx"
},
"source": [
"# Load the Logs"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"id": "zchxB8c7WJe5"
},
"outputs": [],
"source": [
"import pandas as pd"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 81
},
"id": "aKcWr015WNPm",
"outputId": "6e226773-333f-46a2-9fc8-4f54f309d204"
},
"outputs": [
{
"data": {
"text/html": [
"\n",
" <div id=\"df-cd06d09e-fb43-41b0-938f-37f9d285ae66\" class=\"colab-df-container\">\n",
" <div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Success</th>\n",
" <th>Timestamp</th>\n",
" <th>Input</th>\n",
" <th>Output</th>\n",
" <th>RunId (Wandb Runid)</th>\n",
" <th>Model ID (or Name)</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>True</td>\n",
" <td>1694041195</td>\n",
" <td>This is the templated query input</td>\n",
" <td>This is the query output from the model</td>\n",
" <td>8hlumwuk</td>\n",
" <td>OpenAI/Turbo-3.5</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>\n",
" <div class=\"colab-df-buttons\">\n",
"\n",
" <div class=\"colab-df-container\">\n",
" <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-cd06d09e-fb43-41b0-938f-37f9d285ae66')\"\n",
" title=\"Convert this dataframe to an interactive table.\"\n",
" style=\"display:none;\">\n",
"\n",
" <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
" <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
" </svg>\n",
" </button>\n",
"\n",
" <style>\n",
" .colab-df-container {\n",
" display:flex;\n",
" gap: 12px;\n",
" }\n",
"\n",
" .colab-df-convert {\n",
" background-color: #E8F0FE;\n",
" border: none;\n",
" border-radius: 50%;\n",
" cursor: pointer;\n",
" display: none;\n",
" fill: #1967D2;\n",
" height: 32px;\n",
" padding: 0 0 0 0;\n",
" width: 32px;\n",
" }\n",
"\n",
" .colab-df-convert:hover {\n",
" background-color: #E2EBFA;\n",
" box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
" fill: #174EA6;\n",
" }\n",
"\n",
" .colab-df-buttons div {\n",
" margin-bottom: 4px;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert {\n",
" background-color: #3B4455;\n",
" fill: #D2E3FC;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert:hover {\n",
" background-color: #434B5C;\n",
" box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
" filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
" fill: #FFFFFF;\n",
" }\n",
" </style>\n",
"\n",
" <script>\n",
" const buttonEl =\n",
" document.querySelector('#df-cd06d09e-fb43-41b0-938f-37f9d285ae66 button.colab-df-convert');\n",
" buttonEl.style.display =\n",
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
"\n",
" async function convertToInteractive(key) {\n",
" const element = document.querySelector('#df-cd06d09e-fb43-41b0-938f-37f9d285ae66');\n",
" const dataTable =\n",
" await google.colab.kernel.invokeFunction('convertToInteractive',\n",
" [key], {});\n",
" if (!dataTable) return;\n",
"\n",
" const docLinkHtml = 'Like what you see? Visit the ' +\n",
" '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
" + ' to learn more about interactive tables.';\n",
" element.innerHTML = '';\n",
" dataTable['output_type'] = 'display_data';\n",
" await google.colab.output.renderOutput(dataTable, element);\n",
" const docLink = document.createElement('div');\n",
" docLink.innerHTML = docLinkHtml;\n",
" element.appendChild(docLink);\n",
" }\n",
" </script>\n",
" </div>\n",
"\n",
"\n",
" </div>\n",
" </div>\n"
],
"text/plain": [
" Success Timestamp Input \\\n",
"0 True 1694041195 This is the templated query input \n",
"\n",
" Output RunId (Wandb Runid) \\\n",
"0 This is the query output from the model 8hlumwuk \n",
"\n",
" Model ID (or Name) \n",
"0 OpenAI/Turbo-3.5 "
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# path of the csv file\n",
"file_path = 'Model-prompts-example.csv'\n",
"\n",
"# load the csv file as a pandas DataFrame\n",
"data = pd.read_csv(file_path)\n",
"\n",
"data.head()"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"id": "0DbL-kirWUyn"
},
"outputs": [],
"source": [
"input_texts = data['Input'].values"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"id": "cqpAvy8hWXyC"
},
"outputs": [],
"source": [
"messages = [[{\"role\": \"user\", \"content\": input_text}] for input_text in input_texts]"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "SugCyom0Xy8U"
},
"source": [
"# Running Inference"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "qpikx3uxHns3"
},
"outputs": [],
"source": [
"from litellm import batch_completion\n",
"\n",
"model_name = \"facebook/opt-125m\"\n",
"provider = \"vllm\"\n",
"response_list = batch_completion(\n",
" model=model_name,\n",
" custom_llm_provider=provider, # can easily switch to huggingface, replicate, together ai, sagemaker, etc.\n",
" messages=messages,\n",
" temperature=0.2,\n",
" max_tokens=80,\n",
" )"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "QDPikHtwKJJ2",
"outputId": "06f47c44-e258-452a-f9db-232a5b6d2810"
},
"outputs": [
{
"data": {
"text/plain": [
"[<ModelResponse at 0x7e5b87616750> JSON: {\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \".\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is\",\n",
" \"role\": \"assistant\",\n",
" \"logprobs\": null\n",
" }\n",
" }\n",
" ],\n",
" \"created\": 1694053363.6139505,\n",
" \"model\": \"facebook/opt-125m\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": 9,\n",
" \"completion_tokens\": 80,\n",
" \"total_tokens\": 89\n",
" }\n",
" }]"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"response_list"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"id": "SYqTcCiJbQDF"
},
"outputs": [],
"source": [
"response_values = [response['choices'][0]['message']['content'] for response in response_list]"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "wqs-Oy9FbiPo",
"outputId": "16a6a7b7-97c8-4b5b-eff8-09ea5eb5ad06"
},
"outputs": [
{
"data": {
"text/plain": [
"['.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is']"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"response_values"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"id": "mElNbBehbkrz"
},
"outputs": [],
"source": [
"data[f\"{model_name}_output\"] = response_values"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {
"id": "F06NXssDc45k"
},
"outputs": [],
"source": [
"data.to_csv('model_responses.csv', index=False)"
]
}
],
"metadata": {
"accelerator": "GPU",
"colab": {
"gpuType": "V100",
"machine_shape": "hm",
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

7
cookbook/anthropic_agent_sdk/agent_with_mcp.py
View file

@ -7,14 +7,15 @@ with MCP (Model Context Protocol) server integration for enhanced capabilities.
import asyncio
import os
from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions
from claude_agent_sdk import ClaudeAgentOptions, ClaudeSDKClient
from common import (
Config,
fetch_available_models,
setup_litellm_env,
print_header,
handle_model_list,
handle_model_switch,
print_header,
setup_litellm_env,
stream_response,
)

1
cookbook/anthropic_agent_sdk/common.py
View file

@ -3,6 +3,7 @@ Common utilities for Claude Agent SDK examples
"""
import os
import httpx

7
cookbook/anthropic_agent_sdk/main.py
View file

@ -7,14 +7,15 @@ through the Claude Agent SDK by pointing it to the LiteLLM gateway.
"""
import asyncio
from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions
from claude_agent_sdk import ClaudeAgentOptions, ClaudeSDKClient
from common import (
Config,
fetch_available_models,
setup_litellm_env,
print_header,
handle_model_list,
handle_model_switch,
print_header,
setup_litellm_env,
stream_response,
)

7
cookbook/benchmark/benchmark.py
View file

@ -1,11 +1,12 @@
from litellm import completion, completion_cost
import os
import time
import click
from tqdm import tqdm
from tabulate import tabulate
from termcolor import colored
import os
from tqdm import tqdm
from litellm import completion, completion_cost
# Define the list of models to benchmark
# select any LLM listed here: https://docs.litellm.ai/docs/providers

4
cookbook/benchmark/eval_suites_mlflow_autoevals/auto_evals.py
View file

@ -2,10 +2,10 @@ from dotenv import load_dotenv
load_dotenv()
import litellm
from autoevals.llm import *
import litellm
###################
# litellm completion call

20
cookbook/codellama-server/main.py
View file

@ -1,13 +1,15 @@
import traceback
from flask import Flask, request, Response
from flask_cors import CORS
import litellm
from util import handle_error
from litellm import completion
import os
import dotenv
import time
import json
import os
import time
import traceback
import dotenv
from flask import Flask, Response, request
from flask_cors import CORS
from util import handle_error
import litellm
from litellm import completion
dotenv.load_dotenv()

6
cookbook/google_adk_litellm_tutorial.ipynb vendored
View file

@ -73,13 +73,15 @@
"outputs": [],
"source": [
"# Setup environment and API keys\n",
"import os\n",
"import asyncio\n",
"import os\n",
"\n",
"from google.adk.agents import Agent\n",
"from google.adk.models.lite_llm import LiteLlm # For multi-model support\n",
"from google.adk.sessions import InMemorySessionService\n",
"from google.adk.runners import Runner\n",
"from google.adk.sessions import InMemorySessionService\n",
"from google.genai import types\n",
"\n",
"import litellm # Import for proxy configuration\n",
"\n",
"# Set your API keys\n",

489
cookbook/liteLLM_A121_Jurrasic_example.ipynb vendored
View file

@ -1,251 +1,252 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "LeFYo8iqcn5g"
},
"source": [
"# LiteLLM A121 Tutorial\n",
"\n",
"This walks through using A121 Jurassic models\n",
"* j2-light\n",
"* j2-mid\n",
"* j2-ultra"
]
},
"cells": [
{
"cell_type": "markdown",
"source": [
"# LiteLLM A121 Tutorial\n",
"\n",
"This walks through using A121 Jurassic models\n",
"* j2-light\n",
"* j2-mid\n",
"* j2-ultra"
],
"metadata": {
"id": "LeFYo8iqcn5g"
}
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "GslPQFmaZsp-"
},
"outputs": [],
"source": [
"!pip install litellm"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"id": "P3cKiqURZx7P"
},
"outputs": [],
"source": [
"import os\n",
"\n",
"from litellm import completion"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "tmTvA1_GaNU4"
},
"source": [
"## Set A121 Keys\n",
"You can get a free key from https://studio.ai21.com/account/api-key"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"id": "_xX8LmxAZ2vp"
},
"outputs": [],
"source": [
"os.environ[\"AI21_API_KEY\"] = \"\""
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "Fx5ZfJTLbF0A"
},
"source": [
"# A121 Supported Models:\n",
"https://studio.ai21.com/foundation-models"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "H0tl-0Z3bDaL"
},
"source": [
"## J2-light Call"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "DZnApsJUZ_I2",
"outputId": "b5707cbe-f67c-47f7-bac5-a7b8af1ba815"
},
"outputs": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "GslPQFmaZsp-"
},
"outputs": [],
"source": [
"!pip install litellm"
]
},
{
"cell_type": "code",
"source": [
"from litellm import completion\n",
"import os"
],
"metadata": {
"id": "P3cKiqURZx7P"
},
"execution_count": 2,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"## Set A121 Keys\n",
"You can get a free key from https://studio.ai21.com/account/api-key"
],
"metadata": {
"id": "tmTvA1_GaNU4"
}
},
{
"cell_type": "code",
"source": [
"os.environ[\"AI21_API_KEY\"] = \"\""
],
"metadata": {
"id": "_xX8LmxAZ2vp"
},
"execution_count": 5,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"# A121 Supported Models:\n",
"https://studio.ai21.com/foundation-models"
],
"metadata": {
"id": "Fx5ZfJTLbF0A"
}
},
{
"cell_type": "markdown",
"source": [
"## J2-light Call"
],
"metadata": {
"id": "H0tl-0Z3bDaL"
}
},
{
"cell_type": "code",
"source": [
"messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
"response = completion(model=\"j2-light\", messages=messages)\n",
"response"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "DZnApsJUZ_I2",
"outputId": "b5707cbe-f67c-47f7-bac5-a7b8af1ba815"
},
"execution_count": 6,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"<ModelResponse at 0x7b2c2902e610> JSON: {\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \" However, I have an important question to ask you\\nMy name is X, and I was wondering if you would be willing to help me.\",\n",
" \"role\": \"assistant\"\n",
" }\n",
" }\n",
" ],\n",
" \"created\": 1692761063.5189915,\n",
" \"model\": \"j2-light\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": null,\n",
" \"completion_tokens\": null,\n",
" \"total_tokens\": null\n",
" }\n",
"}"
]
},
"metadata": {},
"execution_count": 6
}
]
},
{
"cell_type": "markdown",
"source": [
"# J2-Mid"
],
"metadata": {
"id": "wCcnrYnnbMQA"
}
},
{
"cell_type": "code",
"source": [
"messages = [{ \"content\": \"what model are you\",\"role\": \"user\"}]\n",
"response = completion(model=\"j2-mid\", messages=messages)\n",
"response"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "-5Sxf4blaeEl",
"outputId": "6264a5e8-16d6-44a3-e167-9e0c59b6dbc4"
},
"execution_count": 7,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"<ModelResponse at 0x7b2c2902f6a0> JSON: {\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \"\\nplease choose the model from the list below\\nModel view in Tekla Structures\",\n",
" \"role\": \"assistant\"\n",
" }\n",
" }\n",
" ],\n",
" \"created\": 1692761140.0017524,\n",
" \"model\": \"j2-mid\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": null,\n",
" \"completion_tokens\": null,\n",
" \"total_tokens\": null\n",
" }\n",
"}"
]
},
"metadata": {},
"execution_count": 7
}
]
},
{
"cell_type": "markdown",
"source": [
"# J2-Ultra"
],
"metadata": {
"id": "wDARpjxtbUcg"
}
},
{
"cell_type": "code",
"source": [
"messages = [{ \"content\": \"what model are you\",\"role\": \"user\"}]\n",
"response = completion(model=\"j2-ultra\", messages=messages)\n",
"response"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "i228xwsYbSYo",
"outputId": "3765ac56-5a9b-442e-b357-2e346d02e1df"
},
"execution_count": 8,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"<ModelResponse at 0x7b2c28fd4090> JSON: {\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \"\\nI am not a specific model, but I can provide information and assistance based on my training data. Please let me know if there is anything you\",\n",
" \"role\": \"assistant\"\n",
" }\n",
" }\n",
" ],\n",
" \"created\": 1692761157.8675153,\n",
" \"model\": \"j2-ultra\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": null,\n",
" \"completion_tokens\": null,\n",
" \"total_tokens\": null\n",
" }\n",
"}"
]
},
"metadata": {},
"execution_count": 8
}
"data": {
"text/plain": [
"<ModelResponse at 0x7b2c2902e610> JSON: {\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \" However, I have an important question to ask you\\nMy name is X, and I was wondering if you would be willing to help me.\",\n",
" \"role\": \"assistant\"\n",
" }\n",
" }\n",
" ],\n",
" \"created\": 1692761063.5189915,\n",
" \"model\": \"j2-light\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": null,\n",
" \"completion_tokens\": null,\n",
" \"total_tokens\": null\n",
" }\n",
"}"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
]
],
"source": [
"messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
"response = completion(model=\"j2-light\", messages=messages)\n",
"response"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "wCcnrYnnbMQA"
},
"source": [
"# J2-Mid"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "-5Sxf4blaeEl",
"outputId": "6264a5e8-16d6-44a3-e167-9e0c59b6dbc4"
},
"outputs": [
{
"data": {
"text/plain": [
"<ModelResponse at 0x7b2c2902f6a0> JSON: {\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \"\\nplease choose the model from the list below\\nModel view in Tekla Structures\",\n",
" \"role\": \"assistant\"\n",
" }\n",
" }\n",
" ],\n",
" \"created\": 1692761140.0017524,\n",
" \"model\": \"j2-mid\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": null,\n",
" \"completion_tokens\": null,\n",
" \"total_tokens\": null\n",
" }\n",
"}"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"messages = [{ \"content\": \"what model are you\",\"role\": \"user\"}]\n",
"response = completion(model=\"j2-mid\", messages=messages)\n",
"response"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "wDARpjxtbUcg"
},
"source": [
"# J2-Ultra"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "i228xwsYbSYo",
"outputId": "3765ac56-5a9b-442e-b357-2e346d02e1df"
},
"outputs": [
{
"data": {
"text/plain": [
"<ModelResponse at 0x7b2c28fd4090> JSON: {\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \"\\nI am not a specific model, but I can provide information and assistance based on my training data. Please let me know if there is anything you\",\n",
" \"role\": \"assistant\"\n",
" }\n",
" }\n",
" ],\n",
" \"created\": 1692761157.8675153,\n",
" \"model\": \"j2-ultra\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": null,\n",
" \"completion_tokens\": null,\n",
" \"total_tokens\": null\n",
" }\n",
"}"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"messages = [{ \"content\": \"what model are you\",\"role\": \"user\"}]\n",
"response = completion(model=\"j2-ultra\", messages=messages)\n",
"response"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

1
cookbook/liteLLM_Baseten.ipynb vendored
View file

@ -43,6 +43,7 @@
"outputs": [],
"source": [
"import os\n",
"\n",
"from litellm import completion\n",
"\n",
"# Set your Baseten API key\n",

817
cookbook/liteLLM_Getting_Started.ipynb vendored
View file

@ -1,411 +1,412 @@
{
"cells": [
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "MZ01up0p7wOJ"
},
"source": [
"## 🚅 liteLLM Quick Start Demo\n",
"### TLDR: Call 50+ LLM APIs using chatGPT Input/Output format\n",
"https://github.com/BerriAI/litellm\n",
"\n",
"liteLLM is package to simplify calling **OpenAI, Azure, Llama2, Cohere, Anthropic, Huggingface API Endpoints**. LiteLLM manages\n",
"\n"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "RZtzCnQS7rW-"
},
"source": [
"## Installation and setting Params"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "rsrN5W-N7L8d"
},
"outputs": [],
"source": [
"!pip install litellm"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"id": "ArrWyG5b7QAG"
},
"outputs": [],
"source": [
"from litellm import completion\n",
"import os"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "bbhJRt34_NJ1"
},
"source": [
"## Set your API keys\n",
"- liteLLM reads your .env, env variables or key manager for Auth\n",
"\n",
"Set keys for the models you want to use below"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {
"id": "-h8Ga5cR7SvV"
},
"outputs": [],
"source": [
"# Only set keys for the LLMs you want to use\n",
"os.environ['OPENAI_API_KEY'] = \"\" #@param\n",
"os.environ[\"ANTHROPIC_API_KEY\"] = \"\" #@param\n",
"os.environ[\"REPLICATE_API_KEY\"] = \"\" #@param\n",
"os.environ[\"COHERE_API_KEY\"] = \"\" #@param\n",
"os.environ[\"AZURE_API_BASE\"] = \"\" #@param\n",
"os.environ[\"AZURE_API_VERSION\"] = \"\" #@param\n",
"os.environ[\"AZURE_API_KEY\"] = \"\" #@param"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "fhqpKv6L8fBj"
},
"source": [
"## Call chatGPT"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "speIkoX_8db4",
"outputId": "331a6c65-f121-4e65-e121-bf8aaad05d9d"
},
"outputs": [
{
"data": {
"text/plain": [
"<OpenAIObject chat.completion id=chatcmpl-820kPkRwSLml4X6165fWbZlEDOedr at 0x12ff93630> JSON: {\n",
" \"id\": \"chatcmpl-820kPkRwSLml4X6165fWbZlEDOedr\",\n",
" \"object\": \"chat.completion\",\n",
" \"created\": 1695490221,\n",
" \"model\": \"gpt-3.5-turbo-0613\",\n",
" \"choices\": [\n",
" {\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": \"I'm sorry, but as an AI text-based model, I don't have real-time information. However, you can check the current weather in San Francisco by searching for \\\"weather in SF\\\" on any search engine or checking a weather website or app.\"\n",
" },\n",
" \"finish_reason\": \"stop\"\n",
" }\n",
" ],\n",
" \"usage\": {\n",
" \"prompt_tokens\": 13,\n",
" \"completion_tokens\": 51,\n",
" \"total_tokens\": 64\n",
" },\n",
" \"response_ms\": 2385.592\n",
"}"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"completion(model=\"gpt-3.5-turbo\", messages=[{ \"content\": \"what's the weather in SF\",\"role\": \"user\"}])"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "Q3jV1Uxv8zNo"
},
"source": [
"## Call Claude-2"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "V8yTWYzY8m9S",
"outputId": "8b6dd32d-f9bf-4e89-886d-47cb8020f025"
},
"outputs": [
{
"data": {
"text/plain": [
"<ModelResponse chat.completion id=chatcmpl-6d1a40c0-19c0-4bd7-9ca2-a91d8b8c2295 at 0x12ff85a40> JSON: {\n",
" \"object\": \"chat.completion\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop_sequence\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \" Unfortunately I don't have enough context to know the exact location you are asking about when you say \\\"SF\\\". SF could refer to San Francisco, California, or potentially other cities that go by SF as an abbreviation. To get an accurate weather report, it would be helpful if you could provide the full city name and state/country. If you are looking for the weather in San Francisco, California, I would be happy to provide that forecast. Please let me know the specific location you want the weather for.\",\n",
" \"role\": \"assistant\",\n",
" \"logprobs\": null\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-6d1a40c0-19c0-4bd7-9ca2-a91d8b8c2295\",\n",
" \"created\": 1695490260.983768,\n",
" \"response_ms\": 6351.544,\n",
" \"model\": \"claude-2\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": 14,\n",
" \"completion_tokens\": 102,\n",
" \"total_tokens\": 116\n",
" }\n",
"}"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"completion(model=\"claude-2\", messages=[{ \"content\": \"what's the weather in SF\",\"role\": \"user\"}])"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "yu0LPDmW9PJa"
},
"source": [
"## Call llama2 on replicate"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "0GWV5mtO9Jbu",
"outputId": "38538825-b271-406d-a437-f5cf0eb7e548"
},
"outputs": [
{
"data": {
"text/plain": [
"<ModelResponse chat.completion id=chatcmpl-3151c2eb-b26f-4c96-89b5-ed1746b219e0 at 0x138b87e50> JSON: {\n",
" \"object\": \"chat.completion\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \" I'm happy to help! However, I must point out that the question \\\"what's the weather in SF\\\" doesn't make sense as \\\"SF\\\" could refer to multiple locations. Could you please clarify which location you are referring to? San Francisco, California or Sioux Falls, South Dakota? Once I have more context, I would be happy to provide you with accurate and reliable information.\",\n",
" \"role\": \"assistant\",\n",
" \"logprobs\": null\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-3151c2eb-b26f-4c96-89b5-ed1746b219e0\",\n",
" \"created\": 1695490237.714101,\n",
" \"response_ms\": 12109.565,\n",
" \"model\": \"replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": 6,\n",
" \"completion_tokens\": 78,\n",
" \"total_tokens\": 84\n",
" },\n",
" \"ended\": 1695490249.821266\n",
"}"
]
},
"execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model = \"replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1\"\n",
"completion(model=model, messages=[{ \"content\": \"what's the weather in SF\",\"role\": \"user\"}])"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "HXdj5SEe9iLK"
},
"source": [
"## Call Command-Nightly"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "EaUq2xIx9fhr",
"outputId": "55fe6f52-b58b-4729-948a-74dac4b431b2"
},
"outputs": [
{
"data": {
"text/plain": [
"<ModelResponse chat.completion id=chatcmpl-dc0d8ead-071d-486c-a111-78975b38794b at 0x1389725e0> JSON: {\n",
" \"object\": \"chat.completion\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \" As an AI model I don't have access to real-time data, so I can't tell\",\n",
" \"role\": \"assistant\",\n",
" \"logprobs\": null\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-dc0d8ead-071d-486c-a111-78975b38794b\",\n",
" \"created\": 1695490235.936903,\n",
" \"response_ms\": 1022.6759999999999,\n",
" \"model\": \"command-nightly\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": 6,\n",
" \"completion_tokens\": 19,\n",
" \"total_tokens\": 25\n",
" }\n",
"}"
]
},
"execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"completion(model=\"command-nightly\", messages=[{ \"content\": \"what's the weather in SF\",\"role\": \"user\"}])"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "1g9hSgsL9soJ"
},
"source": [
"## Call Azure OpenAI"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"For azure openai calls ensure to add the `azure/` prefix to `model`. If your deployment-id is `chatgpt-test` set `model` = `azure/chatgpt-test`"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "AvLjR-PF-lt0",
"outputId": "deff2db3-b003-48cd-ea62-c03a68a4464a"
},
"outputs": [
{
"data": {
"text/plain": [
"<OpenAIObject chat.completion id=chatcmpl-820kZyCwbNvZATiLkNmXmpxxzvTKO at 0x138b84ae0> JSON: {\n",
" \"id\": \"chatcmpl-820kZyCwbNvZATiLkNmXmpxxzvTKO\",\n",
" \"object\": \"chat.completion\",\n",
" \"created\": 1695490231,\n",
" \"model\": \"gpt-35-turbo\",\n",
" \"choices\": [\n",
" {\n",
" \"index\": 0,\n",
" \"finish_reason\": \"stop\",\n",
" \"message\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": \"Sorry, as an AI language model, I don't have real-time information. Please check your preferred weather website or app for the latest weather updates of San Francisco.\"\n",
" }\n",
" }\n",
" ],\n",
" \"usage\": {\n",
" \"completion_tokens\": 33,\n",
" \"prompt_tokens\": 14,\n",
" \"total_tokens\": 47\n",
" },\n",
" \"response_ms\": 1499.529\n",
"}"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"completion(model=\"azure/chatgpt-v-2\", messages=[{ \"content\": \"what's the weather in SF\",\"role\": \"user\"}])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.6"
}
"cells": [
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "MZ01up0p7wOJ"
},
"source": [
"## 🚅 liteLLM Quick Start Demo\n",
"### TLDR: Call 50+ LLM APIs using chatGPT Input/Output format\n",
"https://github.com/BerriAI/litellm\n",
"\n",
"liteLLM is package to simplify calling **OpenAI, Azure, Llama2, Cohere, Anthropic, Huggingface API Endpoints**. LiteLLM manages\n",
"\n"
]
},
"nbformat": 4,
"nbformat_minor": 0
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "RZtzCnQS7rW-"
},
"source": [
"## Installation and setting Params"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "rsrN5W-N7L8d"
},
"outputs": [],
"source": [
"!pip install litellm"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"id": "ArrWyG5b7QAG"
},
"outputs": [],
"source": [
"import os\n",
"\n",
"from litellm import completion"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "bbhJRt34_NJ1"
},
"source": [
"## Set your API keys\n",
"- liteLLM reads your .env, env variables or key manager for Auth\n",
"\n",
"Set keys for the models you want to use below"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {
"id": "-h8Ga5cR7SvV"
},
"outputs": [],
"source": [
"# Only set keys for the LLMs you want to use\n",
"os.environ['OPENAI_API_KEY'] = \"\" #@param\n",
"os.environ[\"ANTHROPIC_API_KEY\"] = \"\" #@param\n",
"os.environ[\"REPLICATE_API_KEY\"] = \"\" #@param\n",
"os.environ[\"COHERE_API_KEY\"] = \"\" #@param\n",
"os.environ[\"AZURE_API_BASE\"] = \"\" #@param\n",
"os.environ[\"AZURE_API_VERSION\"] = \"\" #@param\n",
"os.environ[\"AZURE_API_KEY\"] = \"\" #@param"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "fhqpKv6L8fBj"
},
"source": [
"## Call chatGPT"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "speIkoX_8db4",
"outputId": "331a6c65-f121-4e65-e121-bf8aaad05d9d"
},
"outputs": [
{
"data": {
"text/plain": [
"<OpenAIObject chat.completion id=chatcmpl-820kPkRwSLml4X6165fWbZlEDOedr at 0x12ff93630> JSON: {\n",
" \"id\": \"chatcmpl-820kPkRwSLml4X6165fWbZlEDOedr\",\n",
" \"object\": \"chat.completion\",\n",
" \"created\": 1695490221,\n",
" \"model\": \"gpt-3.5-turbo-0613\",\n",
" \"choices\": [\n",
" {\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": \"I'm sorry, but as an AI text-based model, I don't have real-time information. However, you can check the current weather in San Francisco by searching for \\\"weather in SF\\\" on any search engine or checking a weather website or app.\"\n",
" },\n",
" \"finish_reason\": \"stop\"\n",
" }\n",
" ],\n",
" \"usage\": {\n",
" \"prompt_tokens\": 13,\n",
" \"completion_tokens\": 51,\n",
" \"total_tokens\": 64\n",
" },\n",
" \"response_ms\": 2385.592\n",
"}"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"completion(model=\"gpt-3.5-turbo\", messages=[{ \"content\": \"what's the weather in SF\",\"role\": \"user\"}])"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "Q3jV1Uxv8zNo"
},
"source": [
"## Call Claude-2"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "V8yTWYzY8m9S",
"outputId": "8b6dd32d-f9bf-4e89-886d-47cb8020f025"
},
"outputs": [
{
"data": {
"text/plain": [
"<ModelResponse chat.completion id=chatcmpl-6d1a40c0-19c0-4bd7-9ca2-a91d8b8c2295 at 0x12ff85a40> JSON: {\n",
" \"object\": \"chat.completion\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop_sequence\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \" Unfortunately I don't have enough context to know the exact location you are asking about when you say \\\"SF\\\". SF could refer to San Francisco, California, or potentially other cities that go by SF as an abbreviation. To get an accurate weather report, it would be helpful if you could provide the full city name and state/country. If you are looking for the weather in San Francisco, California, I would be happy to provide that forecast. Please let me know the specific location you want the weather for.\",\n",
" \"role\": \"assistant\",\n",
" \"logprobs\": null\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-6d1a40c0-19c0-4bd7-9ca2-a91d8b8c2295\",\n",
" \"created\": 1695490260.983768,\n",
" \"response_ms\": 6351.544,\n",
" \"model\": \"claude-2\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": 14,\n",
" \"completion_tokens\": 102,\n",
" \"total_tokens\": 116\n",
" }\n",
"}"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"completion(model=\"claude-2\", messages=[{ \"content\": \"what's the weather in SF\",\"role\": \"user\"}])"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "yu0LPDmW9PJa"
},
"source": [
"## Call llama2 on replicate"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "0GWV5mtO9Jbu",
"outputId": "38538825-b271-406d-a437-f5cf0eb7e548"
},
"outputs": [
{
"data": {
"text/plain": [
"<ModelResponse chat.completion id=chatcmpl-3151c2eb-b26f-4c96-89b5-ed1746b219e0 at 0x138b87e50> JSON: {\n",
" \"object\": \"chat.completion\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \" I'm happy to help! However, I must point out that the question \\\"what's the weather in SF\\\" doesn't make sense as \\\"SF\\\" could refer to multiple locations. Could you please clarify which location you are referring to? San Francisco, California or Sioux Falls, South Dakota? Once I have more context, I would be happy to provide you with accurate and reliable information.\",\n",
" \"role\": \"assistant\",\n",
" \"logprobs\": null\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-3151c2eb-b26f-4c96-89b5-ed1746b219e0\",\n",
" \"created\": 1695490237.714101,\n",
" \"response_ms\": 12109.565,\n",
" \"model\": \"replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": 6,\n",
" \"completion_tokens\": 78,\n",
" \"total_tokens\": 84\n",
" },\n",
" \"ended\": 1695490249.821266\n",
"}"
]
},
"execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model = \"replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1\"\n",
"completion(model=model, messages=[{ \"content\": \"what's the weather in SF\",\"role\": \"user\"}])"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "HXdj5SEe9iLK"
},
"source": [
"## Call Command-Nightly"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "EaUq2xIx9fhr",
"outputId": "55fe6f52-b58b-4729-948a-74dac4b431b2"
},
"outputs": [
{
"data": {
"text/plain": [
"<ModelResponse chat.completion id=chatcmpl-dc0d8ead-071d-486c-a111-78975b38794b at 0x1389725e0> JSON: {\n",
" \"object\": \"chat.completion\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \" As an AI model I don't have access to real-time data, so I can't tell\",\n",
" \"role\": \"assistant\",\n",
" \"logprobs\": null\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-dc0d8ead-071d-486c-a111-78975b38794b\",\n",
" \"created\": 1695490235.936903,\n",
" \"response_ms\": 1022.6759999999999,\n",
" \"model\": \"command-nightly\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": 6,\n",
" \"completion_tokens\": 19,\n",
" \"total_tokens\": 25\n",
" }\n",
"}"
]
},
"execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"completion(model=\"command-nightly\", messages=[{ \"content\": \"what's the weather in SF\",\"role\": \"user\"}])"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "1g9hSgsL9soJ"
},
"source": [
"## Call Azure OpenAI"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"For azure openai calls ensure to add the `azure/` prefix to `model`. If your deployment-id is `chatgpt-test` set `model` = `azure/chatgpt-test`"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "AvLjR-PF-lt0",
"outputId": "deff2db3-b003-48cd-ea62-c03a68a4464a"
},
"outputs": [
{
"data": {
"text/plain": [
"<OpenAIObject chat.completion id=chatcmpl-820kZyCwbNvZATiLkNmXmpxxzvTKO at 0x138b84ae0> JSON: {\n",
" \"id\": \"chatcmpl-820kZyCwbNvZATiLkNmXmpxxzvTKO\",\n",
" \"object\": \"chat.completion\",\n",
" \"created\": 1695490231,\n",
" \"model\": \"gpt-35-turbo\",\n",
" \"choices\": [\n",
" {\n",
" \"index\": 0,\n",
" \"finish_reason\": \"stop\",\n",
" \"message\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": \"Sorry, as an AI language model, I don't have real-time information. Please check your preferred weather website or app for the latest weather updates of San Francisco.\"\n",
" }\n",
" }\n",
" ],\n",
" \"usage\": {\n",
" \"completion_tokens\": 33,\n",
" \"prompt_tokens\": 14,\n",
" \"total_tokens\": 47\n",
" },\n",
" \"response_ms\": 1499.529\n",
"}"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"completion(model=\"azure/chatgpt-v-2\", messages=[{ \"content\": \"what's the weather in SF\",\"role\": \"user\"}])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.6"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

7
cookbook/liteLLM_IBM_Watsonx.ipynb vendored
View file

@ -39,8 +39,10 @@
"outputs": [],
"source": [
"import os\n",
"\n",
"import litellm\n",
"from litellm.llms.watsonx import IBMWatsonXAI\n",
"\n",
"litellm.set_verbose = False\n",
"\n",
"os.environ[\"WATSONX_URL\"] = \"\" # Your watsonx.ai base URL\n",
@ -178,9 +180,10 @@
}
],
"source": [
"from litellm import acompletion\n",
"import asyncio\n",
"\n",
"from litellm import acompletion\n",
"\n",
"granite_task = acompletion(\n",
" model=\"watsonx/ibm/granite-13b-chat-v2\",\n",
" messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
@ -256,7 +259,7 @@
}
],
"source": [
"from litellm import embedding, aembedding\n",
"from litellm import aembedding, embedding\n",
"\n",
"response = embedding(\n",
" model=\"watsonx/ibm/slate-30m-english-rtrvr\",\n",

1
cookbook/liteLLM_Langchain_Demo.ipynb vendored
View file

@ -41,6 +41,7 @@
"outputs": [],
"source": [
"import os\n",
"\n",
"from langchain.chat_models import ChatLiteLLM\n",
"from langchain.schema import HumanMessage"
]

1
cookbook/liteLLM_Ollama.ipynb vendored
View file

@ -182,6 +182,7 @@
"source": [
"import litellm\n",
"\n",
"\n",
"async def async_ollama():\n",
" response = await litellm.acompletion(\n",
" model=\"ollama/llama2\", \n",

472
cookbook/liteLLM_Replicate_Demo.ipynb vendored
View file

@ -1,238 +1,240 @@
{
"cells": [
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "YV6L5fNv7Kep"
},
"source": [
"# Call Replicate LLMs using chatGPT Input/Output Format\n",
"This tutorial covers using the following Replicate Models with liteLLM\n",
"\n",
"- [StableLM Tuned Alpha 7B](https://replicate.com/stability-ai/stablelm-tuned-alpha-7b)\n",
"- [LLAMA-2 70B Chat](https://replicate.com/replicate/llama-2-70b-chat)\n",
"- [A16z infra-LLAMA-2 7B Chat](https://replicate.com/a16z-infra/llama-2-7b-chat)\n",
"- [Dolly V2 12B](https://replicate.com/replicate/dolly-v2-12b)\n",
"- [Vicuna 13B](https://replicate.com/replicate/vicuna-13b)\n",
"\n",
"\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "TO-EdF84O9QT"
},
"outputs": [],
"source": [
"# install liteLLM\n",
"!pip install litellm"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "mpHTbTqQ8fey"
},
"source": [
"Imports & Set ENV variables\n",
"Get your Replicate Key: https://replicate.com/account/api-tokens"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"id": "kDbgfcU8O-dW"
},
"outputs": [],
"source": [
"from litellm import completion\n",
"import os\n",
"os.environ['REPLICATE_API_TOKEN'] = ' ' # @param\n",
"user_message = \"Hello, whats the weather in San Francisco??\"\n",
"messages = [{ \"content\": user_message,\"role\": \"user\"}]"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "1KmkOdzLSOmJ"
},
"source": [
"## Call Replicate Models using completion(model, messages) - chatGPT format"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "XJ4nh4SnRzHP",
"outputId": "986c0544-bb40-4915-f00f-498b0e518307"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"replicate is not installed. Installing...\n",
"Response from stability-ai/stablelm-tuned-alpha-7b:c49dae362cbaecd2ceabb5bd34fdb68413c4ff775111fea065d259d577757beb \n",
"]\n",
"\n",
"{'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': \"I'm sorry for you being unable to access this content as my training data only goes up until 2023/03. However I can tell you what your local weather forecast may look like at any time of year with respect to current conditions:\"}}], 'created': 1691611730.7224207, 'model': 'stability-ai/stablelm-tuned-alpha-7b:c49dae362cbaecd2ceabb5bd34fdb68413c4ff775111fea065d259d577757beb', 'usage': {'prompt_tokens': 9, 'completion_tokens': 49, 'total_tokens': 58}}\n",
"Response from replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1 \n",
"]\n",
"\n",
"{'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': \" Hello! I'm happy to help you with your question. However, I must point out that the question itself may not be meaningful. San Francisco is a city located in California, USA, and it is not possible for me to provide you with the current weather conditions there as I am a text-based AI language model and do not have access to real-time weather data. Additionally, the weather in San Francisco can vary greatly depending on the time of year, so it would be best to check a reliable weather source for the most up-to-date information.\\n\\nIf you meant to ask a different question, please feel free to rephrase it, and I will do my best to assist you in a safe and positive manner.\"}}], 'created': 1691611745.0269957, 'model': 'replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1', 'usage': {'prompt_tokens': 9, 'completion_tokens': 143, 'total_tokens': 152}}\n",
"Response from a16z-infra/llama-2-7b-chat:4f0b260b6a13eb53a6b1891f089d57c08f41003ae79458be5011303d81a394dc \n",
"]\n",
"\n",
"{'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': \" Hello! I'm here to help you with your question. However, I must inform you that the weather in San Francisco can be quite unpredictable and can change rapidly. It's important to check reliable sources such as AccuWeather or the National Weather Service for the most up-to-date and accurate information about the weather in San Francisco.\\nI cannot provide you with real-time weather data or forecasts as I'm just an AI and do not have access to current weather conditions or predictions. But I can suggest some trustworthy websites or apps where you can find the latest weather updates:\\n* AccuWeather (accuweather.com)\\n* The Weather Channel (weather.com)\\n* Dark Sky (darksky.net)\\n* Weather Underground (wunderground.com)\\nRemember, it's always best to consult multiple sources for the most accurate information when planning your day or trip. Enjoy your day!\"}}], 'created': 1691611748.7723358, 'model': 'a16z-infra/llama-2-7b-chat:4f0b260b6a13eb53a6b1891f089d57c08f41003ae79458be5011303d81a394dc', 'usage': {'prompt_tokens': 9, 'completion_tokens': 174, 'total_tokens': 183}}\n",
"Response from replicate/dolly-v2-12b:ef0e1aefc61f8e096ebe4db6b2bacc297daf2ef6899f0f7e001ec445893500e5 \n",
"]\n",
"\n",
"{'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': 'Its 68 degrees right now in San Francisco! The temperature will be rising through the week and i expect it to reach 70 on Thursdays and Friday. Skies are expected to be partly cloudy with some sun breaks throughout the day.\\n\\n'}}], 'created': 1691611752.2002115, 'model': 'replicate/dolly-v2-12b:ef0e1aefc61f8e096ebe4db6b2bacc297daf2ef6899f0f7e001ec445893500e5', 'usage': {'prompt_tokens': 9, 'completion_tokens': 48, 'total_tokens': 57}}\n",
"Response from replicate/vicuna-13b:6282abe6a492de4145d7bb601023762212f9ddbbe78278bd6771c8b3b2f2a13b \n",
"]\n",
"\n",
"{'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': ''}}], 'created': 1691611752.8998356, 'model': 'replicate/vicuna-13b:6282abe6a492de4145d7bb601023762212f9ddbbe78278bd6771c8b3b2f2a13b', 'usage': {'prompt_tokens': 9, 'completion_tokens': 0, 'total_tokens': 9}}\n"
]
}
],
"source": [
"llama_2 = \"replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1\"\n",
"llama_2_7b = \"a16z-infra/llama-2-7b-chat:4f0b260b6a13eb53a6b1891f089d57c08f41003ae79458be5011303d81a394dc\"\n",
"dolly_v2 = \"replicate/dolly-v2-12b:ef0e1aefc61f8e096ebe4db6b2bacc297daf2ef6899f0f7e001ec445893500e5\"\n",
"vicuna = \"replicate/vicuna-13b:6282abe6a492de4145d7bb601023762212f9ddbbe78278bd6771c8b3b2f2a13b\"\n",
"models = [llama_2, llama_2_7b, dolly_v2, vicuna]\n",
"for model in models:\n",
" response = completion(model=model, messages=messages)\n",
" print(f\"Response from {model} \\n]\\n\")\n",
" print(response)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "zlTVLB-7PTV_",
"outputId": "5182275b-3108-46fa-a2cf-745fac4ad110"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Hi\n",
" there!\n",
" The\n",
" current\n",
" forecast\n",
" for\n",
" today's\n",
" high\n",
" temperature\n",
" ranges\n",
" from\n",
" 75\n",
" degrees\n",
" Fahrenheit\n",
" all\n",
" day\n",
" to\n",
" 83\n",
" degrees\n",
" Fahrenheit\n",
" with\n",
" possible\n",
" isolated\n",
" thunderstorms\n",
" during\n",
" the\n",
" afternoon\n",
" hours,\n",
" mainly\n",
" at\n",
" sunset\n",
" through\n",
" early\n",
" evening. The\n",
" Pacific\n",
" Ocean\n",
" has\n",
" a\n",
" low\n",
" pressure\n",
" of\n",
" 926\n",
" mb\n",
" and\n",
" mostly\n",
" cloud\n",
" cover\n",
" in\n",
" this\n",
" region\n",
" on\n",
" sunny\n",
" days\n",
" due\n",
" to\n",
" warming\n",
" temperatures\n",
" above\n",
" average\n",
" along\n",
" most\n",
" coastal\n",
" areas\n",
" and\n",
" ocean\n",
" breezes.<|USER|>\n"
]
}
],
"source": [
"# @title Stream Responses from Replicate - Outputs in the same format used by chatGPT streaming\n",
"response = completion(model=llama_2, messages=messages, stream=True)\n",
"\n",
"for chunk in response:\n",
" print(chunk['choices'][0]['delta'])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "t7WMRuL-8NrO"
},
"outputs": [],
"source": []
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
"cells": [
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "YV6L5fNv7Kep"
},
"source": [
"# Call Replicate LLMs using chatGPT Input/Output Format\n",
"This tutorial covers using the following Replicate Models with liteLLM\n",
"\n",
"- [StableLM Tuned Alpha 7B](https://replicate.com/stability-ai/stablelm-tuned-alpha-7b)\n",
"- [LLAMA-2 70B Chat](https://replicate.com/replicate/llama-2-70b-chat)\n",
"- [A16z infra-LLAMA-2 7B Chat](https://replicate.com/a16z-infra/llama-2-7b-chat)\n",
"- [Dolly V2 12B](https://replicate.com/replicate/dolly-v2-12b)\n",
"- [Vicuna 13B](https://replicate.com/replicate/vicuna-13b)\n",
"\n",
"\n",
"\n"
]
},
"nbformat": 4,
"nbformat_minor": 0
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "TO-EdF84O9QT"
},
"outputs": [],
"source": [
"# install liteLLM\n",
"!pip install litellm"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "mpHTbTqQ8fey"
},
"source": [
"Imports & Set ENV variables\n",
"Get your Replicate Key: https://replicate.com/account/api-tokens"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"id": "kDbgfcU8O-dW"
},
"outputs": [],
"source": [
"import os\n",
"\n",
"from litellm import completion\n",
"\n",
"os.environ['REPLICATE_API_TOKEN'] = ' ' # @param\n",
"user_message = \"Hello, whats the weather in San Francisco??\"\n",
"messages = [{ \"content\": user_message,\"role\": \"user\"}]"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "1KmkOdzLSOmJ"
},
"source": [
"## Call Replicate Models using completion(model, messages) - chatGPT format"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "XJ4nh4SnRzHP",
"outputId": "986c0544-bb40-4915-f00f-498b0e518307"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"replicate is not installed. Installing...\n",
"Response from stability-ai/stablelm-tuned-alpha-7b:c49dae362cbaecd2ceabb5bd34fdb68413c4ff775111fea065d259d577757beb \n",
"]\n",
"\n",
"{'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': \"I'm sorry for you being unable to access this content as my training data only goes up until 2023/03. However I can tell you what your local weather forecast may look like at any time of year with respect to current conditions:\"}}], 'created': 1691611730.7224207, 'model': 'stability-ai/stablelm-tuned-alpha-7b:c49dae362cbaecd2ceabb5bd34fdb68413c4ff775111fea065d259d577757beb', 'usage': {'prompt_tokens': 9, 'completion_tokens': 49, 'total_tokens': 58}}\n",
"Response from replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1 \n",
"]\n",
"\n",
"{'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': \" Hello! I'm happy to help you with your question. However, I must point out that the question itself may not be meaningful. San Francisco is a city located in California, USA, and it is not possible for me to provide you with the current weather conditions there as I am a text-based AI language model and do not have access to real-time weather data. Additionally, the weather in San Francisco can vary greatly depending on the time of year, so it would be best to check a reliable weather source for the most up-to-date information.\\n\\nIf you meant to ask a different question, please feel free to rephrase it, and I will do my best to assist you in a safe and positive manner.\"}}], 'created': 1691611745.0269957, 'model': 'replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1', 'usage': {'prompt_tokens': 9, 'completion_tokens': 143, 'total_tokens': 152}}\n",
"Response from a16z-infra/llama-2-7b-chat:4f0b260b6a13eb53a6b1891f089d57c08f41003ae79458be5011303d81a394dc \n",
"]\n",
"\n",
"{'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': \" Hello! I'm here to help you with your question. However, I must inform you that the weather in San Francisco can be quite unpredictable and can change rapidly. It's important to check reliable sources such as AccuWeather or the National Weather Service for the most up-to-date and accurate information about the weather in San Francisco.\\nI cannot provide you with real-time weather data or forecasts as I'm just an AI and do not have access to current weather conditions or predictions. But I can suggest some trustworthy websites or apps where you can find the latest weather updates:\\n* AccuWeather (accuweather.com)\\n* The Weather Channel (weather.com)\\n* Dark Sky (darksky.net)\\n* Weather Underground (wunderground.com)\\nRemember, it's always best to consult multiple sources for the most accurate information when planning your day or trip. Enjoy your day!\"}}], 'created': 1691611748.7723358, 'model': 'a16z-infra/llama-2-7b-chat:4f0b260b6a13eb53a6b1891f089d57c08f41003ae79458be5011303d81a394dc', 'usage': {'prompt_tokens': 9, 'completion_tokens': 174, 'total_tokens': 183}}\n",
"Response from replicate/dolly-v2-12b:ef0e1aefc61f8e096ebe4db6b2bacc297daf2ef6899f0f7e001ec445893500e5 \n",
"]\n",
"\n",
"{'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': 'Its 68 degrees right now in San Francisco! The temperature will be rising through the week and i expect it to reach 70 on Thursdays and Friday. Skies are expected to be partly cloudy with some sun breaks throughout the day.\\n\\n'}}], 'created': 1691611752.2002115, 'model': 'replicate/dolly-v2-12b:ef0e1aefc61f8e096ebe4db6b2bacc297daf2ef6899f0f7e001ec445893500e5', 'usage': {'prompt_tokens': 9, 'completion_tokens': 48, 'total_tokens': 57}}\n",
"Response from replicate/vicuna-13b:6282abe6a492de4145d7bb601023762212f9ddbbe78278bd6771c8b3b2f2a13b \n",
"]\n",
"\n",
"{'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': ''}}], 'created': 1691611752.8998356, 'model': 'replicate/vicuna-13b:6282abe6a492de4145d7bb601023762212f9ddbbe78278bd6771c8b3b2f2a13b', 'usage': {'prompt_tokens': 9, 'completion_tokens': 0, 'total_tokens': 9}}\n"
]
}
],
"source": [
"llama_2 = \"replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1\"\n",
"llama_2_7b = \"a16z-infra/llama-2-7b-chat:4f0b260b6a13eb53a6b1891f089d57c08f41003ae79458be5011303d81a394dc\"\n",
"dolly_v2 = \"replicate/dolly-v2-12b:ef0e1aefc61f8e096ebe4db6b2bacc297daf2ef6899f0f7e001ec445893500e5\"\n",
"vicuna = \"replicate/vicuna-13b:6282abe6a492de4145d7bb601023762212f9ddbbe78278bd6771c8b3b2f2a13b\"\n",
"models = [llama_2, llama_2_7b, dolly_v2, vicuna]\n",
"for model in models:\n",
" response = completion(model=model, messages=messages)\n",
" print(f\"Response from {model} \\n]\\n\")\n",
" print(response)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "zlTVLB-7PTV_",
"outputId": "5182275b-3108-46fa-a2cf-745fac4ad110"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Hi\n",
" there!\n",
" The\n",
" current\n",
" forecast\n",
" for\n",
" today's\n",
" high\n",
" temperature\n",
" ranges\n",
" from\n",
" 75\n",
" degrees\n",
" Fahrenheit\n",
" all\n",
" day\n",
" to\n",
" 83\n",
" degrees\n",
" Fahrenheit\n",
" with\n",
" possible\n",
" isolated\n",
" thunderstorms\n",
" during\n",
" the\n",
" afternoon\n",
" hours,\n",
" mainly\n",
" at\n",
" sunset\n",
" through\n",
" early\n",
" evening. The\n",
" Pacific\n",
" Ocean\n",
" has\n",
" a\n",
" low\n",
" pressure\n",
" of\n",
" 926\n",
" mb\n",
" and\n",
" mostly\n",
" cloud\n",
" cover\n",
" in\n",
" this\n",
" region\n",
" on\n",
" sunny\n",
" days\n",
" due\n",
" to\n",
" warming\n",
" temperatures\n",
" above\n",
" average\n",
" along\n",
" most\n",
" coastal\n",
" areas\n",
" and\n",
" ocean\n",
" breezes.<|USER|>\n"
]
}
],
"source": [
"# @title Stream Responses from Replicate - Outputs in the same format used by chatGPT streaming\n",
"response = completion(model=llama_2, messages=messages, stream=True)\n",
"\n",
"for chunk in response:\n",
" print(chunk['choices'][0]['delta'])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "t7WMRuL-8NrO"
},
"outputs": [],
"source": []
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

441
cookbook/liteLLM_Streaming_Demo.ipynb vendored
View file

@ -1,226 +1,227 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "YV6L5fNv7Kep"
},
"source": [
"# [STREAMING] OpenAI, Anthropic, Replicate, Cohere using liteLLM\n",
"In this tutorial:\n",
"Note: All inputs/outputs are in the format used by `gpt-3.5-turbo`\n",
"\n",
"- Call all models in the same input format [**with streaming**]:\n",
"\n",
" `completion(model, messages, stream=True)`\n",
"- All streaming generators are accessed at `chunk['choices'][0]['delta']`\n",
"\n",
"The following Models are covered in this tutorial\n",
"- [GPT-3.5-Turbo](https://platform.openai.com/docs/models/gpt-3-5)\n",
"- [Claude-2](https://www.anthropic.com/index/claude-2)\n",
"- [StableLM Tuned Alpha 7B](https://replicate.com/stability-ai/stablelm-tuned-alpha-7b)\n",
"- [A16z infra-LLAMA-2 7B Chat](https://replicate.com/a16z-infra/llama-2-7b-chat)\n",
"- [Vicuna 13B](https://replicate.com/replicate/vicuna-13b)\n",
"- [Cohere - Command Nightly]()\n",
"\n",
"\n",
"\n"
]
},
"cells": [
{
"cell_type": "markdown",
"source": [
"# [STREAMING] OpenAI, Anthropic, Replicate, Cohere using liteLLM\n",
"In this tutorial:\n",
"Note: All inputs/outputs are in the format used by `gpt-3.5-turbo`\n",
"\n",
"- Call all models in the same input format [**with streaming**]:\n",
"\n",
" `completion(model, messages, stream=True)`\n",
"- All streaming generators are accessed at `chunk['choices'][0]['delta']`\n",
"\n",
"The following Models are covered in this tutorial\n",
"- [GPT-3.5-Turbo](https://platform.openai.com/docs/models/gpt-3-5)\n",
"- [Claude-2](https://www.anthropic.com/index/claude-2)\n",
"- [StableLM Tuned Alpha 7B](https://replicate.com/stability-ai/stablelm-tuned-alpha-7b)\n",
"- [A16z infra-LLAMA-2 7B Chat](https://replicate.com/a16z-infra/llama-2-7b-chat)\n",
"- [Vicuna 13B](https://replicate.com/replicate/vicuna-13b)\n",
"- [Cohere - Command Nightly]()\n",
"\n",
"\n",
"\n"
],
"metadata": {
"id": "YV6L5fNv7Kep"
}
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "TO-EdF84O9QT"
},
"outputs": [],
"source": [
"# install liteLLM\n",
"!pip install litellm==0.1.369"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "mpHTbTqQ8fey"
},
"source": [
"## Imports & Set ENV variables\n",
"Get your API Keys\n",
"\n",
"https://platform.openai.com/account/api-keys\n",
"\n",
"https://replicate.com/account/api-tokens\n",
"\n",
"https://console.anthropic.com/account/keys\n",
"\n",
"https://dashboard.cohere.ai/api-keys\n"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"id": "kDbgfcU8O-dW"
},
"outputs": [],
"source": [
"import os\n",
"\n",
"from litellm import completion\n",
"\n",
"os.environ['OPENAI_API_KEY'] = '' # @param\n",
"os.environ['REPLICATE_API_TOKEN'] = '' # @param\n",
"os.environ['ANTHROPIC_API_KEY'] = '' # @param\n",
"os.environ['COHERE_API_KEY'] = '' # @param"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "1KmkOdzLSOmJ"
},
"source": [
"### Set Messages"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"id": "xIEeOhVH-oh6"
},
"outputs": [],
"source": [
"user_message = \"Hello, whats the weather in San Francisco??\"\n",
"messages = [{ \"content\": user_message,\"role\": \"user\"}]"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "9SOCVRC1L-G3"
},
"source": [
"## Calling Models using liteLLM Streaming -\n",
"\n",
"## `completion(model, messages, stream)`"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "XJ4nh4SnRzHP",
"outputId": "26b9fe10-b499-4a97-d60d-a8cb8f8030b8"
},
"outputs": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "TO-EdF84O9QT"
},
"outputs": [],
"source": [
"# install liteLLM\n",
"!pip install litellm==0.1.369"
]
},
{
"cell_type": "markdown",
"source": [
"## Imports & Set ENV variables\n",
"Get your API Keys\n",
"\n",
"https://platform.openai.com/account/api-keys\n",
"\n",
"https://replicate.com/account/api-tokens\n",
"\n",
"https://console.anthropic.com/account/keys\n",
"\n",
"https://dashboard.cohere.ai/api-keys\n"
],
"metadata": {
"id": "mpHTbTqQ8fey"
}
},
{
"cell_type": "code",
"source": [
"from litellm import completion\n",
"import os\n",
"\n",
"os.environ['OPENAI_API_KEY'] = '' # @param\n",
"os.environ['REPLICATE_API_TOKEN'] = '' # @param\n",
"os.environ['ANTHROPIC_API_KEY'] = '' # @param\n",
"os.environ['COHERE_API_KEY'] = '' # @param"
],
"metadata": {
"id": "kDbgfcU8O-dW"
},
"execution_count": 8,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"### Set Messages"
],
"metadata": {
"id": "1KmkOdzLSOmJ"
}
},
{
"cell_type": "code",
"source": [
"user_message = \"Hello, whats the weather in San Francisco??\"\n",
"messages = [{ \"content\": user_message,\"role\": \"user\"}]"
],
"metadata": {
"id": "xIEeOhVH-oh6"
},
"execution_count": 4,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"## Calling Models using liteLLM Streaming -\n",
"\n",
"## `completion(model, messages, stream)`"
],
"metadata": {
"id": "9SOCVRC1L-G3"
}
},
{
"cell_type": "code",
"source": [
"# replicate models #######\n",
"stability_ai = \"stability-ai/stablelm-tuned-alpha-7b:c49dae362cbaecd2ceabb5bd34fdb68413c4ff775111fea065d259d577757beb\"\n",
"llama_2_7b = \"a16z-infra/llama-2-7b-chat:4f0b260b6a13eb53a6b1891f089d57c08f41003ae79458be5011303d81a394dc\"\n",
"vicuna = \"replicate/vicuna-13b:6282abe6a492de4145d7bb601023762212f9ddbbe78278bd6771c8b3b2f2a13b\"\n",
"\n",
"models = [\"gpt-3.5-turbo\", \"claude-2\", stability_ai, llama_2_7b, vicuna, \"command-nightly\"] # command-nightly is Cohere\n",
"for model in models:\n",
" replicate = (model == stability_ai or model==llama_2_7b or model==vicuna) # let liteLLM know if a model is replicate, using this optional param, `replicate=True`\n",
" response = completion(model=model, messages=messages, stream=True, replicate=replicate)\n",
" print(f\"####################\\n\\nResponse from {model}\")\n",
" for i, chunk in enumerate(response):\n",
" if i < 5: # NOTE: LIMITING CHUNKS FOR THIS DEMO\n",
" print((chunk['choices'][0]['delta']))\n"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "XJ4nh4SnRzHP",
"outputId": "26b9fe10-b499-4a97-d60d-a8cb8f8030b8"
},
"execution_count": 13,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"####################\n",
"\n",
"Response from gpt-3.5-turbo\n",
"{\n",
" \"role\": \"assistant\",\n",
" \"content\": \"\"\n",
"}\n",
"{\n",
" \"content\": \"I\"\n",
"}\n",
"{\n",
" \"content\": \"'m\"\n",
"}\n",
"{\n",
" \"content\": \" sorry\"\n",
"}\n",
"{\n",
" \"content\": \",\"\n",
"}\n",
"####################\n",
"\n",
"Response from claude-2\n",
"{'role': 'assistant', 'content': ' Unfortunately'}\n",
"{'role': 'assistant', 'content': ' I'}\n",
"{'role': 'assistant', 'content': ' don'}\n",
"{'role': 'assistant', 'content': \"'t\"}\n",
"{'role': 'assistant', 'content': ' have'}\n",
"####################\n",
"\n",
"Response from stability-ai/stablelm-tuned-alpha-7b:c49dae362cbaecd2ceabb5bd34fdb68413c4ff775111fea065d259d577757beb\n",
"{'role': 'assistant', 'content': \"I'm\"}\n",
"{'role': 'assistant', 'content': ' sorry,'}\n",
"{'role': 'assistant', 'content': ' I'}\n",
"{'role': 'assistant', 'content': ' cannot'}\n",
"{'role': 'assistant', 'content': ' answer'}\n",
"####################\n",
"\n",
"Response from a16z-infra/llama-2-7b-chat:4f0b260b6a13eb53a6b1891f089d57c08f41003ae79458be5011303d81a394dc\n",
"{'role': 'assistant', 'content': ''}\n",
"{'role': 'assistant', 'content': ' Hello'}\n",
"{'role': 'assistant', 'content': '!'}\n",
"{'role': 'assistant', 'content': ' I'}\n",
"{'role': 'assistant', 'content': \"'\"}\n",
"####################\n",
"\n",
"Response from replicate/vicuna-13b:6282abe6a492de4145d7bb601023762212f9ddbbe78278bd6771c8b3b2f2a13b\n",
"{'role': 'assistant', 'content': 'Comment:'}\n",
"{'role': 'assistant', 'content': 'Hi! '}\n",
"{'role': 'assistant', 'content': 'How '}\n",
"{'role': 'assistant', 'content': 'are '}\n",
"{'role': 'assistant', 'content': 'you '}\n",
"####################\n",
"\n",
"Response from command-nightly\n",
"{'role': 'assistant', 'content': ' Hello'}\n",
"{'role': 'assistant', 'content': '!'}\n",
"{'role': 'assistant', 'content': ' '}\n",
"{'role': 'assistant', 'content': ' I'}\n",
"{'role': 'assistant', 'content': \"'m\"}\n"
]
}
]
},
{
"cell_type": "code",
"source": [],
"metadata": {
"id": "t7WMRuL-8NrO"
},
"execution_count": null,
"outputs": []
"name": "stdout",
"output_type": "stream",
"text": [
"####################\n",
"\n",
"Response from gpt-3.5-turbo\n",
"{\n",
" \"role\": \"assistant\",\n",
" \"content\": \"\"\n",
"}\n",
"{\n",
" \"content\": \"I\"\n",
"}\n",
"{\n",
" \"content\": \"'m\"\n",
"}\n",
"{\n",
" \"content\": \" sorry\"\n",
"}\n",
"{\n",
" \"content\": \",\"\n",
"}\n",
"####################\n",
"\n",
"Response from claude-2\n",
"{'role': 'assistant', 'content': ' Unfortunately'}\n",
"{'role': 'assistant', 'content': ' I'}\n",
"{'role': 'assistant', 'content': ' don'}\n",
"{'role': 'assistant', 'content': \"'t\"}\n",
"{'role': 'assistant', 'content': ' have'}\n",
"####################\n",
"\n",
"Response from stability-ai/stablelm-tuned-alpha-7b:c49dae362cbaecd2ceabb5bd34fdb68413c4ff775111fea065d259d577757beb\n",
"{'role': 'assistant', 'content': \"I'm\"}\n",
"{'role': 'assistant', 'content': ' sorry,'}\n",
"{'role': 'assistant', 'content': ' I'}\n",
"{'role': 'assistant', 'content': ' cannot'}\n",
"{'role': 'assistant', 'content': ' answer'}\n",
"####################\n",
"\n",
"Response from a16z-infra/llama-2-7b-chat:4f0b260b6a13eb53a6b1891f089d57c08f41003ae79458be5011303d81a394dc\n",
"{'role': 'assistant', 'content': ''}\n",
"{'role': 'assistant', 'content': ' Hello'}\n",
"{'role': 'assistant', 'content': '!'}\n",
"{'role': 'assistant', 'content': ' I'}\n",
"{'role': 'assistant', 'content': \"'\"}\n",
"####################\n",
"\n",
"Response from replicate/vicuna-13b:6282abe6a492de4145d7bb601023762212f9ddbbe78278bd6771c8b3b2f2a13b\n",
"{'role': 'assistant', 'content': 'Comment:'}\n",
"{'role': 'assistant', 'content': 'Hi! '}\n",
"{'role': 'assistant', 'content': 'How '}\n",
"{'role': 'assistant', 'content': 'are '}\n",
"{'role': 'assistant', 'content': 'you '}\n",
"####################\n",
"\n",
"Response from command-nightly\n",
"{'role': 'assistant', 'content': ' Hello'}\n",
"{'role': 'assistant', 'content': '!'}\n",
"{'role': 'assistant', 'content': ' '}\n",
"{'role': 'assistant', 'content': ' I'}\n",
"{'role': 'assistant', 'content': \"'m\"}\n"
]
}
]
],
"source": [
"# replicate models #######\n",
"stability_ai = \"stability-ai/stablelm-tuned-alpha-7b:c49dae362cbaecd2ceabb5bd34fdb68413c4ff775111fea065d259d577757beb\"\n",
"llama_2_7b = \"a16z-infra/llama-2-7b-chat:4f0b260b6a13eb53a6b1891f089d57c08f41003ae79458be5011303d81a394dc\"\n",
"vicuna = \"replicate/vicuna-13b:6282abe6a492de4145d7bb601023762212f9ddbbe78278bd6771c8b3b2f2a13b\"\n",
"\n",
"models = [\"gpt-3.5-turbo\", \"claude-2\", stability_ai, llama_2_7b, vicuna, \"command-nightly\"] # command-nightly is Cohere\n",
"for model in models:\n",
" replicate = (model == stability_ai or model==llama_2_7b or model==vicuna) # let liteLLM know if a model is replicate, using this optional param, `replicate=True`\n",
" response = completion(model=model, messages=messages, stream=True, replicate=replicate)\n",
" print(f\"####################\\n\\nResponse from {model}\")\n",
" for i, chunk in enumerate(response):\n",
" if i < 5: # NOTE: LIMITING CHUNKS FOR THIS DEMO\n",
" print((chunk['choices'][0]['delta']))\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "t7WMRuL-8NrO"
},
"outputs": [],
"source": []
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

1
cookbook/liteLLM_clarifai_Demo.ipynb vendored
View file

@ -41,6 +41,7 @@
"source": [
"## Set Clarifai Credentials\n",
"import os\n",
"\n",
"os.environ[\"CLARIFAI_API_KEY\"]= \"YOUR_CLARIFAI_PAT\" # Clarifai PAT"
]
},

2
cookbook/liteLLM_function_calling.ipynb vendored
View file

@ -33,6 +33,7 @@
"outputs": [],
"source": [
"import os\n",
"\n",
"from litellm import completion"
]
},
@ -215,6 +216,7 @@
],
"source": [
"import json\n",
"\n",
"function_name = function_call_data['name']\n",
"function_args = function_call_data['arguments']\n",
"function_args = json.loads(function_args)\n",

1
cookbook/litellm_Test_Multiple_Providers.ipynb vendored
View file

@ -47,6 +47,7 @@
"outputs": [],
"source": [
"from dotenv import load_dotenv\n",
"\n",
"load_dotenv()"
]
},

5
cookbook/litellm_proxy_server/braintrust_prompt_wrapper_server.py
View file

@ -18,10 +18,9 @@ import os
from typing import Any, Dict, List, Optional
import httpx
from fastapi import FastAPI, HTTPException, Header, Query
from fastapi.responses import JSONResponse
import uvicorn
from fastapi import FastAPI, Header, HTTPException, Query
from fastapi.responses import JSONResponse
app = FastAPI(
title="Braintrust Prompt Wrapper",

1
cookbook/litellm_proxy_server/cli_token_usage.py
View file

@ -7,6 +7,7 @@ in your Python scripts after running `litellm-proxy login`.
"""
from textwrap import indent
import litellm
LITELLM_BASE_URL = "http://localhost:4000/"

5
cookbook/litellm_router/load_test_proxy.py
View file

@ -1,5 +1,6 @@
import sys
import os
import sys
from dotenv import load_dotenv
load_dotenv()
@ -8,8 +9,8 @@ sys.path.insert(
0, os.path.abspath("../..")
) # Adds the parent directory to the system path
from litellm import Router
import litellm
from litellm import Router
litellm.set_verbose = False
os.environ.pop("AZURE_AD_TOKEN")

5
cookbook/litellm_router/load_test_queuing.py
View file

@ -1,5 +1,6 @@
import sys
import os
import sys
from dotenv import load_dotenv
load_dotenv()
@ -8,8 +9,8 @@ sys.path.insert(
0, os.path.abspath("../..")
) # Adds the parent directory to the system path
from litellm import Router
import litellm
from litellm import Router
litellm.set_verbose = False
# os.environ.pop("AZURE_AD_TOKEN")

5
cookbook/litellm_router/load_test_router.py
View file

@ -1,5 +1,6 @@
import sys
import os
import sys
from dotenv import load_dotenv
load_dotenv()
@ -8,8 +9,8 @@ sys.path.insert(
0, os.path.abspath("../..")
) # Adds the parent directory to the system path
from litellm import Router
import litellm
from litellm import Router
litellm.set_verbose = False
os.environ.pop("AZURE_AD_TOKEN")

10
cookbook/litellm_router_load_test/memory_usage/router_endpoint.py
View file

@ -1,10 +1,12 @@
from fastapi import FastAPI
import uvicorn
from memory_profiler import profile
import os
import uvicorn
from dotenv import load_dotenv
from fastapi import FastAPI
from memory_profiler import profile
import litellm
from litellm import Router
from dotenv import load_dotenv
from litellm._uuid import uuid
load_dotenv()

12
cookbook/litellm_router_load_test/memory_usage/router_memory_usage copy.py
View file

@ -1,17 +1,19 @@
#### What this tests ####
from memory_profiler import profile
import sys
import os
import time
import asyncio
import os
import sys
import time
from memory_profiler import profile
sys.path.insert(
0, os.path.abspath("../..")
) # Adds the parent directory to the system path
from dotenv import load_dotenv
import litellm
from litellm import Router
from dotenv import load_dotenv
from litellm._uuid import uuid
load_dotenv()

12
cookbook/litellm_router_load_test/memory_usage/router_memory_usage.py
View file

@ -1,17 +1,19 @@
#### What this tests ####
from memory_profiler import profile
import sys
import os
import time
import asyncio
import os
import sys
import time
from memory_profiler import profile
sys.path.insert(
0, os.path.abspath("../..")
) # Adds the parent directory to the system path
from dotenv import load_dotenv
import litellm
from litellm import Router
from dotenv import load_dotenv
from litellm._uuid import uuid
load_dotenv()

3
cookbook/litellm_router_load_test/memory_usage/send_request.py
View file

@ -1,6 +1,7 @@
import requests
from concurrent.futures import ThreadPoolExecutor
import requests
# Replace the URL with your actual endpoint
url = "http://localhost:8000/router_acompletion"

7
cookbook/litellm_router_load_test/test_loadtest_openai_client.py
View file

@ -1,5 +1,6 @@
import sys
import os
import sys
from dotenv import load_dotenv
load_dotenv()
@ -7,10 +8,12 @@ sys.path.insert(
0, os.path.abspath("../..")
) # Adds the parent directory to the system path
import asyncio
from litellm import Timeout
import time
import openai
from litellm import Timeout
### Test just calling AsyncAzureOpenAI
openai_client = openai.AsyncAzureOpenAI(

6
cookbook/litellm_router_load_test/test_loadtest_router.py
View file

@ -1,5 +1,6 @@
import sys
import os
import sys
from dotenv import load_dotenv
load_dotenv()
@ -7,9 +8,10 @@ sys.path.insert(
0, os.path.abspath("../..")
) # Adds the parent directory to the system path
import asyncio
from litellm import Router, Timeout
import time
from litellm import Router, Timeout
### Test calling router async

8
cookbook/litellm_router_load_test/test_loadtest_router_withs3_cache.py
View file

@ -1,5 +1,6 @@
import sys
import os
import sys
from dotenv import load_dotenv
load_dotenv()
@ -7,10 +8,11 @@ sys.path.insert(
0, os.path.abspath("../..")
) # Adds the parent directory to the system path
import asyncio
from litellm import Router, Timeout
import time
from litellm.caching.caching import Cache
import litellm
from litellm import Router, Timeout
from litellm.caching.caching import Cache
litellm.cache = Cache(
type="s3", s3_bucket_name="cache-bucket-litellm", s3_region_name="us-west-2"

1
cookbook/livekit_agent_sdk/main.py
View file

@ -9,6 +9,7 @@ and Azure realtime APIs without changing your agent code.
import asyncio
import json
import os
import websockets
# Configuration

339
cookbook/logging_observability/LiteLLM_Arize.ipynb vendored
View file

@ -1,172 +1,173 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "4FbDOmcj2VkM"
},
"source": [
"## Use LiteLLM with Arize\n",
"https://docs.litellm.ai/docs/observability/arize_integration\n",
"\n",
"This method uses the litellm proxy to send the data to Arize. The callback is set in the litellm config below, instead of using OpenInference tracing."
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "21W8Woog26Ns"
},
"source": [
"## Install Dependencies"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"id": "xrjKLBxhxu2L"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Requirement already satisfied: litellm in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (1.54.1)\n",
"Requirement already satisfied: aiohttp in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (3.11.10)\n",
"Requirement already satisfied: click in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (8.1.7)\n",
"Requirement already satisfied: httpx<0.28.0,>=0.23.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (0.27.2)\n",
"Requirement already satisfied: importlib-metadata>=6.8.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (8.5.0)\n",
"Requirement already satisfied: jinja2<4.0.0,>=3.1.2 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (3.1.4)\n",
"Requirement already satisfied: jsonschema<5.0.0,>=4.22.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (4.23.0)\n",
"Requirement already satisfied: openai>=1.55.3 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (1.57.1)\n",
"Requirement already satisfied: pydantic<3.0.0,>=2.0.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (2.10.3)\n",
"Requirement already satisfied: python-dotenv>=0.2.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (1.0.1)\n",
"Requirement already satisfied: requests<3.0.0,>=2.31.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (2.32.3)\n",
"Requirement already satisfied: tiktoken>=0.7.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (0.7.0)\n",
"Requirement already satisfied: tokenizers in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (0.21.0)\n",
"Requirement already satisfied: anyio in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from httpx<0.28.0,>=0.23.0->litellm) (4.7.0)\n",
"Requirement already satisfied: certifi in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from httpx<0.28.0,>=0.23.0->litellm) (2024.8.30)\n",
"Requirement already satisfied: httpcore==1.* in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from httpx<0.28.0,>=0.23.0->litellm) (1.0.7)\n",
"Requirement already satisfied: idna in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from httpx<0.28.0,>=0.23.0->litellm) (3.10)\n",
"Requirement already satisfied: sniffio in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from httpx<0.28.0,>=0.23.0->litellm) (1.3.1)\n",
"Requirement already satisfied: h11<0.15,>=0.13 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from httpcore==1.*->httpx<0.28.0,>=0.23.0->litellm) (0.14.0)\n",
"Requirement already satisfied: zipp>=3.20 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from importlib-metadata>=6.8.0->litellm) (3.21.0)\n",
"Requirement already satisfied: MarkupSafe>=2.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from jinja2<4.0.0,>=3.1.2->litellm) (3.0.2)\n",
"Requirement already satisfied: attrs>=22.2.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (24.2.0)\n",
"Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (2024.10.1)\n",
"Requirement already satisfied: referencing>=0.28.4 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (0.35.1)\n",
"Requirement already satisfied: rpds-py>=0.7.1 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (0.22.3)\n",
"Requirement already satisfied: distro<2,>=1.7.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from openai>=1.55.3->litellm) (1.9.0)\n",
"Requirement already satisfied: jiter<1,>=0.4.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from openai>=1.55.3->litellm) (0.6.1)\n",
"Requirement already satisfied: tqdm>4 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from openai>=1.55.3->litellm) (4.67.1)\n",
"Requirement already satisfied: typing-extensions<5,>=4.11 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from openai>=1.55.3->litellm) (4.12.2)\n",
"Requirement already satisfied: annotated-types>=0.6.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.0.0->litellm) (0.7.0)\n",
"Requirement already satisfied: pydantic-core==2.27.1 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.0.0->litellm) (2.27.1)\n",
"Requirement already satisfied: charset-normalizer<4,>=2 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from requests<3.0.0,>=2.31.0->litellm) (3.4.0)\n",
"Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from requests<3.0.0,>=2.31.0->litellm) (2.0.7)\n",
"Requirement already satisfied: regex>=2022.1.18 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from tiktoken>=0.7.0->litellm) (2024.11.6)\n",
"Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from aiohttp->litellm) (2.4.4)\n",
"Requirement already satisfied: aiosignal>=1.1.2 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from aiohttp->litellm) (1.3.1)\n",
"Requirement already satisfied: frozenlist>=1.1.1 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from aiohttp->litellm) (1.5.0)\n",
"Requirement already satisfied: multidict<7.0,>=4.5 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from aiohttp->litellm) (6.1.0)\n",
"Requirement already satisfied: propcache>=0.2.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from aiohttp->litellm) (0.2.1)\n",
"Requirement already satisfied: yarl<2.0,>=1.17.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from aiohttp->litellm) (1.18.3)\n",
"Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from tokenizers->litellm) (0.26.5)\n",
"Requirement already satisfied: filelock in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers->litellm) (3.16.1)\n",
"Requirement already satisfied: fsspec>=2023.5.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers->litellm) (2024.10.0)\n",
"Requirement already satisfied: packaging>=20.9 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers->litellm) (24.2)\n",
"Requirement already satisfied: pyyaml>=5.1 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers->litellm) (6.0.2)\n"
]
}
],
"source": [
"!pip install litellm"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "jHEu-TjZ29PJ"
},
"source": [
"## Set Env Variables"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"id": "QWd9rTysxsWO"
},
"outputs": [],
"source": [
"import litellm\n",
"import os\n",
"from getpass import getpass\n",
"\n",
"os.environ[\"ARIZE_SPACE_KEY\"] = getpass(\"Enter your Arize space key: \")\n",
"os.environ[\"ARIZE_API_KEY\"] = getpass(\"Enter your Arize API key: \")\n",
"os.environ['OPENAI_API_KEY']= getpass(\"Enter your OpenAI API key: \")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Let's run a completion call and see the traces in Arize"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Hello! Nice to meet you, OpenAI. How can I assist you today?\n"
]
}
],
"source": [
"# set arize as a callback, litellm will send the data to arize\n",
"litellm.callbacks = [\"arize\"]\n",
" \n",
"# openai call\n",
"response = litellm.completion(\n",
" model=\"gpt-3.5-turbo\",\n",
" messages=[\n",
" {\"role\": \"user\", \"content\": \"Hi 👋 - i'm openai\"}\n",
" ]\n",
")\n",
"print(response.choices[0].message.content)"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": ".venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.6"
}
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "4FbDOmcj2VkM"
},
"source": [
"## Use LiteLLM with Arize\n",
"https://docs.litellm.ai/docs/observability/arize_integration\n",
"\n",
"This method uses the litellm proxy to send the data to Arize. The callback is set in the litellm config below, instead of using OpenInference tracing."
]
},
"nbformat": 4,
"nbformat_minor": 0
{
"cell_type": "markdown",
"metadata": {
"id": "21W8Woog26Ns"
},
"source": [
"## Install Dependencies"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"id": "xrjKLBxhxu2L"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Requirement already satisfied: litellm in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (1.54.1)\n",
"Requirement already satisfied: aiohttp in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (3.11.10)\n",
"Requirement already satisfied: click in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (8.1.7)\n",
"Requirement already satisfied: httpx<0.28.0,>=0.23.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (0.27.2)\n",
"Requirement already satisfied: importlib-metadata>=6.8.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (8.5.0)\n",
"Requirement already satisfied: jinja2<4.0.0,>=3.1.2 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (3.1.4)\n",
"Requirement already satisfied: jsonschema<5.0.0,>=4.22.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (4.23.0)\n",
"Requirement already satisfied: openai>=1.55.3 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (1.57.1)\n",
"Requirement already satisfied: pydantic<3.0.0,>=2.0.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (2.10.3)\n",
"Requirement already satisfied: python-dotenv>=0.2.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (1.0.1)\n",
"Requirement already satisfied: requests<3.0.0,>=2.31.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (2.32.3)\n",
"Requirement already satisfied: tiktoken>=0.7.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (0.7.0)\n",
"Requirement already satisfied: tokenizers in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (0.21.0)\n",
"Requirement already satisfied: anyio in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from httpx<0.28.0,>=0.23.0->litellm) (4.7.0)\n",
"Requirement already satisfied: certifi in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from httpx<0.28.0,>=0.23.0->litellm) (2024.8.30)\n",
"Requirement already satisfied: httpcore==1.* in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from httpx<0.28.0,>=0.23.0->litellm) (1.0.7)\n",
"Requirement already satisfied: idna in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from httpx<0.28.0,>=0.23.0->litellm) (3.10)\n",
"Requirement already satisfied: sniffio in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from httpx<0.28.0,>=0.23.0->litellm) (1.3.1)\n",
"Requirement already satisfied: h11<0.15,>=0.13 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from httpcore==1.*->httpx<0.28.0,>=0.23.0->litellm) (0.14.0)\n",
"Requirement already satisfied: zipp>=3.20 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from importlib-metadata>=6.8.0->litellm) (3.21.0)\n",
"Requirement already satisfied: MarkupSafe>=2.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from jinja2<4.0.0,>=3.1.2->litellm) (3.0.2)\n",
"Requirement already satisfied: attrs>=22.2.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (24.2.0)\n",
"Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (2024.10.1)\n",
"Requirement already satisfied: referencing>=0.28.4 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (0.35.1)\n",
"Requirement already satisfied: rpds-py>=0.7.1 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (0.22.3)\n",
"Requirement already satisfied: distro<2,>=1.7.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from openai>=1.55.3->litellm) (1.9.0)\n",
"Requirement already satisfied: jiter<1,>=0.4.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from openai>=1.55.3->litellm) (0.6.1)\n",
"Requirement already satisfied: tqdm>4 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from openai>=1.55.3->litellm) (4.67.1)\n",
"Requirement already satisfied: typing-extensions<5,>=4.11 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from openai>=1.55.3->litellm) (4.12.2)\n",
"Requirement already satisfied: annotated-types>=0.6.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.0.0->litellm) (0.7.0)\n",
"Requirement already satisfied: pydantic-core==2.27.1 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.0.0->litellm) (2.27.1)\n",
"Requirement already satisfied: charset-normalizer<4,>=2 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from requests<3.0.0,>=2.31.0->litellm) (3.4.0)\n",
"Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from requests<3.0.0,>=2.31.0->litellm) (2.0.7)\n",
"Requirement already satisfied: regex>=2022.1.18 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from tiktoken>=0.7.0->litellm) (2024.11.6)\n",
"Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from aiohttp->litellm) (2.4.4)\n",
"Requirement already satisfied: aiosignal>=1.1.2 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from aiohttp->litellm) (1.3.1)\n",
"Requirement already satisfied: frozenlist>=1.1.1 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from aiohttp->litellm) (1.5.0)\n",
"Requirement already satisfied: multidict<7.0,>=4.5 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from aiohttp->litellm) (6.1.0)\n",
"Requirement already satisfied: propcache>=0.2.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from aiohttp->litellm) (0.2.1)\n",
"Requirement already satisfied: yarl<2.0,>=1.17.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from aiohttp->litellm) (1.18.3)\n",
"Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from tokenizers->litellm) (0.26.5)\n",
"Requirement already satisfied: filelock in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers->litellm) (3.16.1)\n",
"Requirement already satisfied: fsspec>=2023.5.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers->litellm) (2024.10.0)\n",
"Requirement already satisfied: packaging>=20.9 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers->litellm) (24.2)\n",
"Requirement already satisfied: pyyaml>=5.1 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers->litellm) (6.0.2)\n"
]
}
],
"source": [
"!pip install litellm"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "jHEu-TjZ29PJ"
},
"source": [
"## Set Env Variables"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"id": "QWd9rTysxsWO"
},
"outputs": [],
"source": [
"import os\n",
"from getpass import getpass\n",
"\n",
"import litellm\n",
"\n",
"os.environ[\"ARIZE_SPACE_KEY\"] = getpass(\"Enter your Arize space key: \")\n",
"os.environ[\"ARIZE_API_KEY\"] = getpass(\"Enter your Arize API key: \")\n",
"os.environ['OPENAI_API_KEY']= getpass(\"Enter your OpenAI API key: \")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Let's run a completion call and see the traces in Arize"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Hello! Nice to meet you, OpenAI. How can I assist you today?\n"
]
}
],
"source": [
"# set arize as a callback, litellm will send the data to arize\n",
"litellm.callbacks = [\"arize\"]\n",
" \n",
"# openai call\n",
"response = litellm.completion(\n",
" model=\"gpt-3.5-turbo\",\n",
" messages=[\n",
" {\"role\": \"user\", \"content\": \"Hi 👋 - i'm openai\"}\n",
" ]\n",
")\n",
"print(response.choices[0].message.content)"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": ".venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.6"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

383
cookbook/logging_observability/LiteLLM_Langfuse.ipynb vendored
View file

@ -1,197 +1,198 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "4FbDOmcj2VkM"
},
"source": [
"## Use LiteLLM with Langfuse\n",
"https://docs.litellm.ai/docs/observability/langfuse_integration"
]
},
"cells": [
{
"cell_type": "markdown",
"source": [
"## Use LiteLLM with Langfuse\n",
"https://docs.litellm.ai/docs/observability/langfuse_integration"
],
"metadata": {
"id": "4FbDOmcj2VkM"
}
{
"cell_type": "markdown",
"metadata": {
"id": "21W8Woog26Ns"
},
"source": [
"## Install Dependencies"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "xrjKLBxhxu2L"
},
"outputs": [],
"source": [
"!pip install litellm langfuse"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "jHEu-TjZ29PJ"
},
"source": [
"## Set Env Variables"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"id": "QWd9rTysxsWO"
},
"outputs": [],
"source": [
"import os\n",
"\n",
"import litellm\n",
"from litellm import completion\n",
"\n",
"# from https://cloud.langfuse.com/\n",
"os.environ[\"LANGFUSE_PUBLIC_KEY\"] = \"\"\n",
"os.environ[\"LANGFUSE_SECRET_KEY\"] = \"\"\n",
"\n",
"\n",
"# OpenAI and Cohere keys\n",
"# You can use any of the litellm supported providers: https://docs.litellm.ai/docs/providers\n",
"os.environ['OPENAI_API_KEY']=\"\"\n",
"os.environ['COHERE_API_KEY']=\"\"\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "NodQl0hp3Lma"
},
"source": [
"## Set LangFuse as a callback for sending data\n",
"## OpenAI completion call"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "vNAuwJY1yp_F",
"outputId": "c3a71e26-13f5-4379-fac9-409290ba79bb"
},
"outputs": [
{
"cell_type": "markdown",
"source": [
"## Install Dependencies"
],
"metadata": {
"id": "21W8Woog26Ns"
}
},
{
"cell_type": "code",
"source": [
"!pip install litellm langfuse"
],
"metadata": {
"id": "xrjKLBxhxu2L"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"## Set Env Variables"
],
"metadata": {
"id": "jHEu-TjZ29PJ"
}
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"id": "QWd9rTysxsWO"
},
"outputs": [],
"source": [
"import litellm\n",
"from litellm import completion\n",
"import os\n",
"\n",
"# from https://cloud.langfuse.com/\n",
"os.environ[\"LANGFUSE_PUBLIC_KEY\"] = \"\"\n",
"os.environ[\"LANGFUSE_SECRET_KEY\"] = \"\"\n",
"\n",
"\n",
"# OpenAI and Cohere keys\n",
"# You can use any of the litellm supported providers: https://docs.litellm.ai/docs/providers\n",
"os.environ['OPENAI_API_KEY']=\"\"\n",
"os.environ['COHERE_API_KEY']=\"\"\n"
]
},
{
"cell_type": "markdown",
"source": [
"## Set LangFuse as a callback for sending data\n",
"## OpenAI completion call"
],
"metadata": {
"id": "NodQl0hp3Lma"
}
},
{
"cell_type": "code",
"source": [
"# set langfuse as a callback, litellm will send the data to langfuse\n",
"litellm.success_callback = [\"langfuse\"]\n",
"\n",
"# openai call\n",
"response = completion(\n",
" model=\"gpt-3.5-turbo\",\n",
" messages=[\n",
" {\"role\": \"user\", \"content\": \"Hi 👋 - i'm openai\"}\n",
" ]\n",
")\n",
"\n",
"print(response)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "vNAuwJY1yp_F",
"outputId": "c3a71e26-13f5-4379-fac9-409290ba79bb"
},
"execution_count": 8,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"{\n",
" \"id\": \"chatcmpl-85nP4xHdAP3jAcGneIguWATS9qdoO\",\n",
" \"object\": \"chat.completion\",\n",
" \"created\": 1696392238,\n",
" \"model\": \"gpt-3.5-turbo-0613\",\n",
" \"choices\": [\n",
" {\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": \"Hello! How can I assist you today?\"\n",
" },\n",
" \"finish_reason\": \"stop\"\n",
" }\n",
" ],\n",
" \"usage\": {\n",
" \"prompt_tokens\": 15,\n",
" \"completion_tokens\": 9,\n",
" \"total_tokens\": 24\n",
" }\n",
"}\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"# we set langfuse as a callback in the prev cell\n",
"# cohere call\n",
"response = completion(\n",
" model=\"command-nightly\",\n",
" messages=[\n",
" {\"role\": \"user\", \"content\": \"Hi 👋 - i'm cohere\"}\n",
" ]\n",
")\n",
"\n",
"print(response)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "2PMSLc_FziJL",
"outputId": "1c37605e-b406-4ffc-aafd-e1983489c6be"
},
"execution_count": 9,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"{\n",
" \"object\": \"chat.completion\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \" Nice to meet you, Cohere! I'm excited to be meeting new members of the AI community\",\n",
" \"role\": \"assistant\",\n",
" \"logprobs\": null\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-a14e903f-4608-4ceb-b996-8ebdf21360ca\",\n",
" \"created\": 1696392247.3313863,\n",
" \"model\": \"command-nightly\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": 8,\n",
" \"completion_tokens\": 20,\n",
" \"total_tokens\": 28\n",
" }\n",
"}\n"
]
}
]
"name": "stdout",
"output_type": "stream",
"text": [
"{\n",
" \"id\": \"chatcmpl-85nP4xHdAP3jAcGneIguWATS9qdoO\",\n",
" \"object\": \"chat.completion\",\n",
" \"created\": 1696392238,\n",
" \"model\": \"gpt-3.5-turbo-0613\",\n",
" \"choices\": [\n",
" {\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": \"Hello! How can I assist you today?\"\n",
" },\n",
" \"finish_reason\": \"stop\"\n",
" }\n",
" ],\n",
" \"usage\": {\n",
" \"prompt_tokens\": 15,\n",
" \"completion_tokens\": 9,\n",
" \"total_tokens\": 24\n",
" }\n",
"}\n"
]
}
]
],
"source": [
"# set langfuse as a callback, litellm will send the data to langfuse\n",
"litellm.success_callback = [\"langfuse\"]\n",
"\n",
"# openai call\n",
"response = completion(\n",
" model=\"gpt-3.5-turbo\",\n",
" messages=[\n",
" {\"role\": \"user\", \"content\": \"Hi 👋 - i'm openai\"}\n",
" ]\n",
")\n",
"\n",
"print(response)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "2PMSLc_FziJL",
"outputId": "1c37605e-b406-4ffc-aafd-e1983489c6be"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{\n",
" \"object\": \"chat.completion\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \" Nice to meet you, Cohere! I'm excited to be meeting new members of the AI community\",\n",
" \"role\": \"assistant\",\n",
" \"logprobs\": null\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-a14e903f-4608-4ceb-b996-8ebdf21360ca\",\n",
" \"created\": 1696392247.3313863,\n",
" \"model\": \"command-nightly\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": 8,\n",
" \"completion_tokens\": 20,\n",
" \"total_tokens\": 28\n",
" }\n",
"}\n"
]
}
],
"source": [
"# we set langfuse as a callback in the prev cell\n",
"# cohere call\n",
"response = completion(\n",
" model=\"command-nightly\",\n",
" messages=[\n",
" {\"role\": \"user\", \"content\": \"Hi 👋 - i'm cohere\"}\n",
" ]\n",
")\n",
"\n",
"print(response)"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

692
cookbook/logging_observability/LiteLLM_Lunary.ipynb vendored
View file

@ -1,348 +1,350 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "4FbDOmcj2VkM"
},
"source": [
"## Use LiteLLM with Langfuse\n",
"https://docs.litellm.ai/docs/observability/langfuse_integration"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "21W8Woog26Ns"
},
"source": [
"## Install Dependencies"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "xrjKLBxhxu2L"
},
"outputs": [],
"source": [
"%pip install litellm lunary"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "jHEu-TjZ29PJ"
},
"source": [
"## Set Env Variables"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"id": "QWd9rTysxsWO"
},
"outputs": [],
"source": [
"import litellm\n",
"from litellm import completion\n",
"import os\n",
"\n",
"# from https://app.lunary.ai/\n",
"os.environ[\"LUNARY_PUBLIC_KEY\"] = \"\"\n",
"\n",
"\n",
"# LLM provider keys\n",
"# You can use any of the litellm supported providers: https://docs.litellm.ai/docs/providers\n",
"os.environ['OPENAI_API_KEY'] = \"\"\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "NodQl0hp3Lma"
},
"source": [
"## Set Lunary as a callback for sending data\n",
"## OpenAI completion call"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "vNAuwJY1yp_F",
"outputId": "c3a71e26-13f5-4379-fac9-409290ba79bb"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[Choices(finish_reason='stop', index=0, message=Message(content='Hello! How can I assist you today?', role='assistant'))]ModelResponse(id='chatcmpl-8xIWykI0GiJSmYtXYuB8Z363kpIBm', choices=[Choices(finish_reason='stop', index=0, message=Message(content='Hello! How can I assist you today?', role='assistant'))], created=1709143276, model='gpt-3.5-turbo-0125', object='chat.completion', system_fingerprint='fp_86156a94a0', usage=Usage(completion_tokens=9, prompt_tokens=15, total_tokens=24))\n",
"\n",
"[Lunary] Add event: {\n",
" \"event\": \"start\",\n",
" \"type\": \"llm\",\n",
" \"name\": \"gpt-3.5-turbo\",\n",
" \"runId\": \"a363776a-bd07-4474-bce2-193067f01b2e\",\n",
" \"timestamp\": \"2024-02-28T18:01:15.188153+00:00\",\n",
" \"input\": {\n",
" \"role\": \"user\",\n",
" \"content\": \"Hi \\ud83d\\udc4b - i'm openai\"\n",
" },\n",
" \"extra\": {},\n",
" \"runtime\": \"litellm\",\n",
" \"metadata\": {}\n",
"}\n",
"\n",
"\n",
"[Lunary] Add event: {\n",
" \"event\": \"end\",\n",
" \"type\": \"llm\",\n",
" \"runId\": \"a363776a-bd07-4474-bce2-193067f01b2e\",\n",
" \"timestamp\": \"2024-02-28T18:01:16.846581+00:00\",\n",
" \"output\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": \"Hello! How can I assist you today?\"\n",
" },\n",
" \"runtime\": \"litellm\",\n",
" \"tokensUsage\": {\n",
" \"completion\": 9,\n",
" \"prompt\": 15\n",
" }\n",
"}\n",
"\n",
"\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"--- Logging error ---\n",
"Traceback (most recent call last):\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 537, in _make_request\n",
" response = conn.getresponse()\n",
" ^^^^^^^^^^^^^^^^^^\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connection.py\", line 466, in getresponse\n",
" httplib_response = super().getresponse()\n",
" ^^^^^^^^^^^^^^^^^^^^^\n",
" File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/http/client.py\", line 1423, in getresponse\n",
" response.begin()\n",
" File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/http/client.py\", line 331, in begin\n",
" version, status, reason = self._read_status()\n",
" ^^^^^^^^^^^^^^^^^^^\n",
" File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/http/client.py\", line 292, in _read_status\n",
" line = str(self.fp.readline(_MAXLINE + 1), \"iso-8859-1\")\n",
" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
" File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/socket.py\", line 707, in readinto\n",
" return self._sock.recv_into(b)\n",
" ^^^^^^^^^^^^^^^^^^^^^^^\n",
"TimeoutError: timed out\n",
"\n",
"The above exception was the direct cause of the following exception:\n",
"\n",
"Traceback (most recent call last):\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/adapters.py\", line 486, in send\n",
" resp = conn.urlopen(\n",
" ^^^^^^^^^^^^^\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 847, in urlopen\n",
" retries = retries.increment(\n",
" ^^^^^^^^^^^^^^^^^^\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/util/retry.py\", line 470, in increment\n",
" raise reraise(type(error), error, _stacktrace)\n",
" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/util/util.py\", line 39, in reraise\n",
" raise value\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 793, in urlopen\n",
" response = self._make_request(\n",
" ^^^^^^^^^^^^^^^^^^^\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 539, in _make_request\n",
" self._raise_timeout(err=e, url=url, timeout_value=read_timeout)\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 370, in _raise_timeout\n",
" raise ReadTimeoutError(\n",
"urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='localhost', port=3333): Read timed out. (read timeout=5)\n",
"\n",
"During handling of the above exception, another exception occurred:\n",
"\n",
"Traceback (most recent call last):\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/lunary/consumer.py\", line 59, in send_batch\n",
" response = requests.post(\n",
" ^^^^^^^^^^^^^^\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/api.py\", line 115, in post\n",
" return request(\"post\", url, data=data, json=json, **kwargs)\n",
" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/api.py\", line 59, in request\n",
" return session.request(method=method, url=url, **kwargs)\n",
" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/sessions.py\", line 589, in request\n",
" resp = self.send(prep, **send_kwargs)\n",
" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/sessions.py\", line 703, in send\n",
" r = adapter.send(request, **kwargs)\n",
" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/adapters.py\", line 532, in send\n",
" raise ReadTimeout(e, request=request)\n",
"requests.exceptions.ReadTimeout: HTTPConnectionPool(host='localhost', port=3333): Read timed out. (read timeout=5)\n",
"\n",
"During handling of the above exception, another exception occurred:\n",
"\n",
"Traceback (most recent call last):\n",
" File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/__init__.py\", line 1160, in emit\n",
" msg = self.format(record)\n",
" ^^^^^^^^^^^^^^^^^^^\n",
" File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/__init__.py\", line 999, in format\n",
" return fmt.format(record)\n",
" ^^^^^^^^^^^^^^^^^^\n",
" File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/__init__.py\", line 703, in format\n",
" record.message = record.getMessage()\n",
" ^^^^^^^^^^^^^^^^^^^\n",
" File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/__init__.py\", line 392, in getMessage\n",
" msg = msg % self.args\n",
" ~~~~^~~~~~~~~~~\n",
"TypeError: not all arguments converted during string formatting\n",
"Call stack:\n",
" File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py\", line 1030, in _bootstrap\n",
" self._bootstrap_inner()\n",
" File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py\", line 1073, in _bootstrap_inner\n",
" self.run()\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/lunary/consumer.py\", line 24, in run\n",
" self.send_batch()\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/lunary/consumer.py\", line 73, in send_batch\n",
" logging.error(\"[Lunary] Error sending events\", e)\n",
"Message: '[Lunary] Error sending events'\n",
"Arguments: (ReadTimeout(ReadTimeoutError(\"HTTPConnectionPool(host='localhost', port=3333): Read timed out. (read timeout=5)\")),)\n"
]
}
],
"source": [
"# set langfuse as a callback, litellm will send the data to langfuse\n",
"litellm.success_callback = [\"lunary\"]\n",
"\n",
"# openai call\n",
"response = completion(\n",
" model=\"gpt-3.5-turbo\",\n",
" messages=[\n",
" {\"role\": \"user\", \"content\": \"Hi 👋 - i'm openai\"}\n",
" ]\n",
")\n",
"\n",
"print(response)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Using LiteLLM with Lunary Templates\n",
"\n",
"You can use LiteLLM seamlessly with Lunary templates to manage your prompts and completions.\n",
"\n",
"Assuming you have created a template \"test-template\" with a variable \"question\", you can use it like this:"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "2PMSLc_FziJL",
"outputId": "1c37605e-b406-4ffc-aafd-e1983489c6be"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[Choices(finish_reason='stop', index=0, message=Message(content='Hello! How can I assist you today?', role='assistant'))]ModelResponse(id='chatcmpl-8xIXegwpudg4YKnLB6pmpFGXqTHcH', choices=[Choices(finish_reason='stop', index=0, message=Message(content='Hello! How can I assist you today?', role='assistant'))], created=1709143318, model='gpt-4-0125-preview', object='chat.completion', system_fingerprint='fp_c8aa5a06d6', usage=Usage(completion_tokens=9, prompt_tokens=21, total_tokens=30))\n",
"\n",
"[Lunary] Add event: {\n",
" \"event\": \"start\",\n",
" \"type\": \"llm\",\n",
" \"name\": \"gpt-4-turbo-preview\",\n",
" \"runId\": \"3a5b698d-cb55-4b3b-ab6d-04d2b99e40cb\",\n",
" \"timestamp\": \"2024-02-28T18:01:56.746249+00:00\",\n",
" \"input\": [\n",
" {\n",
" \"role\": \"system\",\n",
" \"content\": \"You are an helpful assistant.\"\n",
" },\n",
" {\n",
" \"role\": \"user\",\n",
" \"content\": \"Hi! Hello!\"\n",
" }\n",
" ],\n",
" \"extra\": {\n",
" \"temperature\": 1,\n",
" \"max_tokens\": 100\n",
" },\n",
" \"runtime\": \"litellm\",\n",
" \"metadata\": {}\n",
"}\n",
"\n",
"\n",
"[Lunary] Add event: {\n",
" \"event\": \"end\",\n",
" \"type\": \"llm\",\n",
" \"runId\": \"3a5b698d-cb55-4b3b-ab6d-04d2b99e40cb\",\n",
" \"timestamp\": \"2024-02-28T18:01:58.741244+00:00\",\n",
" \"output\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": \"Hello! How can I assist you today?\"\n",
" },\n",
" \"runtime\": \"litellm\",\n",
" \"tokensUsage\": {\n",
" \"completion\": 9,\n",
" \"prompt\": 21\n",
" }\n",
"}\n",
"\n",
"\n"
]
}
],
"source": [
"import lunary\n",
"from litellm import completion\n",
"\n",
"template = lunary.render_template(\"test-template\", {\"question\": \"Hello!\"})\n",
"\n",
"response = completion(**template)\n",
"\n",
"print(response)"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.2"
}
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "4FbDOmcj2VkM"
},
"source": [
"## Use LiteLLM with Langfuse\n",
"https://docs.litellm.ai/docs/observability/langfuse_integration"
]
},
"nbformat": 4,
"nbformat_minor": 0
{
"cell_type": "markdown",
"metadata": {
"id": "21W8Woog26Ns"
},
"source": [
"## Install Dependencies"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "xrjKLBxhxu2L"
},
"outputs": [],
"source": [
"%pip install litellm lunary"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "jHEu-TjZ29PJ"
},
"source": [
"## Set Env Variables"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"id": "QWd9rTysxsWO"
},
"outputs": [],
"source": [
"import os\n",
"\n",
"import litellm\n",
"from litellm import completion\n",
"\n",
"# from https://app.lunary.ai/\n",
"os.environ[\"LUNARY_PUBLIC_KEY\"] = \"\"\n",
"\n",
"\n",
"# LLM provider keys\n",
"# You can use any of the litellm supported providers: https://docs.litellm.ai/docs/providers\n",
"os.environ['OPENAI_API_KEY'] = \"\"\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "NodQl0hp3Lma"
},
"source": [
"## Set Lunary as a callback for sending data\n",
"## OpenAI completion call"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "vNAuwJY1yp_F",
"outputId": "c3a71e26-13f5-4379-fac9-409290ba79bb"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[Choices(finish_reason='stop', index=0, message=Message(content='Hello! How can I assist you today?', role='assistant'))]ModelResponse(id='chatcmpl-8xIWykI0GiJSmYtXYuB8Z363kpIBm', choices=[Choices(finish_reason='stop', index=0, message=Message(content='Hello! How can I assist you today?', role='assistant'))], created=1709143276, model='gpt-3.5-turbo-0125', object='chat.completion', system_fingerprint='fp_86156a94a0', usage=Usage(completion_tokens=9, prompt_tokens=15, total_tokens=24))\n",
"\n",
"[Lunary] Add event: {\n",
" \"event\": \"start\",\n",
" \"type\": \"llm\",\n",
" \"name\": \"gpt-3.5-turbo\",\n",
" \"runId\": \"a363776a-bd07-4474-bce2-193067f01b2e\",\n",
" \"timestamp\": \"2024-02-28T18:01:15.188153+00:00\",\n",
" \"input\": {\n",
" \"role\": \"user\",\n",
" \"content\": \"Hi \\ud83d\\udc4b - i'm openai\"\n",
" },\n",
" \"extra\": {},\n",
" \"runtime\": \"litellm\",\n",
" \"metadata\": {}\n",
"}\n",
"\n",
"\n",
"[Lunary] Add event: {\n",
" \"event\": \"end\",\n",
" \"type\": \"llm\",\n",
" \"runId\": \"a363776a-bd07-4474-bce2-193067f01b2e\",\n",
" \"timestamp\": \"2024-02-28T18:01:16.846581+00:00\",\n",
" \"output\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": \"Hello! How can I assist you today?\"\n",
" },\n",
" \"runtime\": \"litellm\",\n",
" \"tokensUsage\": {\n",
" \"completion\": 9,\n",
" \"prompt\": 15\n",
" }\n",
"}\n",
"\n",
"\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"--- Logging error ---\n",
"Traceback (most recent call last):\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 537, in _make_request\n",
" response = conn.getresponse()\n",
" ^^^^^^^^^^^^^^^^^^\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connection.py\", line 466, in getresponse\n",
" httplib_response = super().getresponse()\n",
" ^^^^^^^^^^^^^^^^^^^^^\n",
" File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/http/client.py\", line 1423, in getresponse\n",
" response.begin()\n",
" File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/http/client.py\", line 331, in begin\n",
" version, status, reason = self._read_status()\n",
" ^^^^^^^^^^^^^^^^^^^\n",
" File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/http/client.py\", line 292, in _read_status\n",
" line = str(self.fp.readline(_MAXLINE + 1), \"iso-8859-1\")\n",
" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
" File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/socket.py\", line 707, in readinto\n",
" return self._sock.recv_into(b)\n",
" ^^^^^^^^^^^^^^^^^^^^^^^\n",
"TimeoutError: timed out\n",
"\n",
"The above exception was the direct cause of the following exception:\n",
"\n",
"Traceback (most recent call last):\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/adapters.py\", line 486, in send\n",
" resp = conn.urlopen(\n",
" ^^^^^^^^^^^^^\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 847, in urlopen\n",
" retries = retries.increment(\n",
" ^^^^^^^^^^^^^^^^^^\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/util/retry.py\", line 470, in increment\n",
" raise reraise(type(error), error, _stacktrace)\n",
" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/util/util.py\", line 39, in reraise\n",
" raise value\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 793, in urlopen\n",
" response = self._make_request(\n",
" ^^^^^^^^^^^^^^^^^^^\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 539, in _make_request\n",
" self._raise_timeout(err=e, url=url, timeout_value=read_timeout)\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 370, in _raise_timeout\n",
" raise ReadTimeoutError(\n",
"urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='localhost', port=3333): Read timed out. (read timeout=5)\n",
"\n",
"During handling of the above exception, another exception occurred:\n",
"\n",
"Traceback (most recent call last):\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/lunary/consumer.py\", line 59, in send_batch\n",
" response = requests.post(\n",
" ^^^^^^^^^^^^^^\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/api.py\", line 115, in post\n",
" return request(\"post\", url, data=data, json=json, **kwargs)\n",
" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/api.py\", line 59, in request\n",
" return session.request(method=method, url=url, **kwargs)\n",
" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/sessions.py\", line 589, in request\n",
" resp = self.send(prep, **send_kwargs)\n",
" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/sessions.py\", line 703, in send\n",
" r = adapter.send(request, **kwargs)\n",
" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/adapters.py\", line 532, in send\n",
" raise ReadTimeout(e, request=request)\n",
"requests.exceptions.ReadTimeout: HTTPConnectionPool(host='localhost', port=3333): Read timed out. (read timeout=5)\n",
"\n",
"During handling of the above exception, another exception occurred:\n",
"\n",
"Traceback (most recent call last):\n",
" File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/__init__.py\", line 1160, in emit\n",
" msg = self.format(record)\n",
" ^^^^^^^^^^^^^^^^^^^\n",
" File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/__init__.py\", line 999, in format\n",
" return fmt.format(record)\n",
" ^^^^^^^^^^^^^^^^^^\n",
" File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/__init__.py\", line 703, in format\n",
" record.message = record.getMessage()\n",
" ^^^^^^^^^^^^^^^^^^^\n",
" File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/__init__.py\", line 392, in getMessage\n",
" msg = msg % self.args\n",
" ~~~~^~~~~~~~~~~\n",
"TypeError: not all arguments converted during string formatting\n",
"Call stack:\n",
" File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py\", line 1030, in _bootstrap\n",
" self._bootstrap_inner()\n",
" File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py\", line 1073, in _bootstrap_inner\n",
" self.run()\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/lunary/consumer.py\", line 24, in run\n",
" self.send_batch()\n",
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/lunary/consumer.py\", line 73, in send_batch\n",
" logging.error(\"[Lunary] Error sending events\", e)\n",
"Message: '[Lunary] Error sending events'\n",
"Arguments: (ReadTimeout(ReadTimeoutError(\"HTTPConnectionPool(host='localhost', port=3333): Read timed out. (read timeout=5)\")),)\n"
]
}
],
"source": [
"# set langfuse as a callback, litellm will send the data to langfuse\n",
"litellm.success_callback = [\"lunary\"]\n",
"\n",
"# openai call\n",
"response = completion(\n",
" model=\"gpt-3.5-turbo\",\n",
" messages=[\n",
" {\"role\": \"user\", \"content\": \"Hi 👋 - i'm openai\"}\n",
" ]\n",
")\n",
"\n",
"print(response)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Using LiteLLM with Lunary Templates\n",
"\n",
"You can use LiteLLM seamlessly with Lunary templates to manage your prompts and completions.\n",
"\n",
"Assuming you have created a template \"test-template\" with a variable \"question\", you can use it like this:"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "2PMSLc_FziJL",
"outputId": "1c37605e-b406-4ffc-aafd-e1983489c6be"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[Choices(finish_reason='stop', index=0, message=Message(content='Hello! How can I assist you today?', role='assistant'))]ModelResponse(id='chatcmpl-8xIXegwpudg4YKnLB6pmpFGXqTHcH', choices=[Choices(finish_reason='stop', index=0, message=Message(content='Hello! How can I assist you today?', role='assistant'))], created=1709143318, model='gpt-4-0125-preview', object='chat.completion', system_fingerprint='fp_c8aa5a06d6', usage=Usage(completion_tokens=9, prompt_tokens=21, total_tokens=30))\n",
"\n",
"[Lunary] Add event: {\n",
" \"event\": \"start\",\n",
" \"type\": \"llm\",\n",
" \"name\": \"gpt-4-turbo-preview\",\n",
" \"runId\": \"3a5b698d-cb55-4b3b-ab6d-04d2b99e40cb\",\n",
" \"timestamp\": \"2024-02-28T18:01:56.746249+00:00\",\n",
" \"input\": [\n",
" {\n",
" \"role\": \"system\",\n",
" \"content\": \"You are an helpful assistant.\"\n",
" },\n",
" {\n",
" \"role\": \"user\",\n",
" \"content\": \"Hi! Hello!\"\n",
" }\n",
" ],\n",
" \"extra\": {\n",
" \"temperature\": 1,\n",
" \"max_tokens\": 100\n",
" },\n",
" \"runtime\": \"litellm\",\n",
" \"metadata\": {}\n",
"}\n",
"\n",
"\n",
"[Lunary] Add event: {\n",
" \"event\": \"end\",\n",
" \"type\": \"llm\",\n",
" \"runId\": \"3a5b698d-cb55-4b3b-ab6d-04d2b99e40cb\",\n",
" \"timestamp\": \"2024-02-28T18:01:58.741244+00:00\",\n",
" \"output\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": \"Hello! How can I assist you today?\"\n",
" },\n",
" \"runtime\": \"litellm\",\n",
" \"tokensUsage\": {\n",
" \"completion\": 9,\n",
" \"prompt\": 21\n",
" }\n",
"}\n",
"\n",
"\n"
]
}
],
"source": [
"import lunary\n",
"\n",
"from litellm import completion\n",
"\n",
"template = lunary.render_template(\"test-template\", {\"question\": \"Hello!\"})\n",
"\n",
"response = completion(**template)\n",
"\n",
"print(response)"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.2"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

3
cookbook/logging_observability/LiteLLM_Proxy_Langfuse.ipynb vendored
View file

@ -106,6 +106,7 @@
],
"source": [
"import openai\n",
"\n",
"client = openai.OpenAI(\n",
" api_key=LITELLM_VIRTUAL_KEY,\n",
" base_url=LITELLM_PROXY_BASE_URL\n",
@ -161,6 +162,7 @@
],
"source": [
"import openai\n",
"\n",
"client = openai.OpenAI(\n",
" api_key=LITELLM_VIRTUAL_KEY,\n",
" base_url=LITELLM_PROXY_BASE_URL\n",
@ -208,6 +210,7 @@
],
"source": [
"import openai\n",
"\n",
"client = openai.OpenAI(\n",
" api_key=LITELLM_VIRTUAL_KEY,\n",
" base_url=LITELLM_PROXY_BASE_URL\n",

2
cookbook/misc/migrate_proxy_config.py
View file

@ -12,8 +12,8 @@ Step 2: reads `model_list` and loops through all models
Step 3: calls `<proxy-base-url>/model/new` for each model
"""
import yaml
import requests
import yaml
_in_memory_os_variables = {}

7
cookbook/misc/openai_timeouts.py
View file

@ -1,7 +1,8 @@
import os
from openai import OpenAI
from dotenv import load_dotenv
import concurrent.futures
import os
from dotenv import load_dotenv
from openai import OpenAI
load_dotenv()

6
cookbook/misc/sagmaker_streaming.py
View file

@ -1,9 +1,9 @@
# Notes - on how to do sagemaker streaming using boto3
import json
import boto3
import sys
import os
import sys
import boto3
from dotenv import load_dotenv
load_dotenv()

3
cookbook/misc/test_responses_api.py
View file

@ -1,7 +1,8 @@
import base64
from openai import OpenAI
import time
from openai import OpenAI
client = OpenAI(base_url="http://0.0.0.0:4001", api_key="sk-1234")

3
cookbook/mlflow_langchain_tracing_litellm_proxy.ipynb vendored
View file

@ -85,8 +85,9 @@
},
"outputs": [],
"source": [
"import mlflow\n",
"from operator import itemgetter\n",
"\n",
"import mlflow\n",
"from langchain_core.output_parsers import StrOutputParser\n",
"from langchain_core.prompts import PromptTemplate\n",
"from langchain_core.runnables import RunnableLambda\n",

4
cookbook/mock_prompt_management_server/mock_prompt_management_server.py
View file

@ -14,11 +14,11 @@ Test the endpoint:
curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt"
"""
import os
import json
import os
from typing import Any, Dict, List, Optional
from fastapi import FastAPI, HTTPException, Header, Query, status
from fastapi import FastAPI, Header, HTTPException, Query, status
from fastapi.responses import JSONResponse
from pydantic import BaseModel, Field

3
cookbook/nova_sonic_realtime.py
View file

@ -17,9 +17,10 @@ import asyncio
import base64
import json
import os
from typing import Optional
import pyaudio
import websockets
from typing import Optional
# Bounded queue size for audio chunks (configurable via env to avoid unbounded memory)
AUDIO_QUEUE_MAXSIZE = int(os.getenv("LITELLM_ASYNCIO_QUEUE_MAXSIZE", 10_000))

3
cookbook/veo_video_generation.py
View file

@ -15,9 +15,10 @@ Requirements:
import json
import os
import time
import requests
from typing import Optional
import requests
class VeoVideoGenerator:
"""Complete Veo video generation client using LiteLLM proxy."""

1
db_scripts/create_views.py
View file

@ -5,7 +5,6 @@ python script to pre-create all views required by LiteLLM Proxy Server
import asyncio
# Enter your DATABASE_URL here
from prisma import Prisma
db = Prisma(

10
db_scripts/migrate_keys.py
View file

@ -1,11 +1,11 @@
from prisma import Prisma
import asyncio
import csv
import json
import asyncio
from datetime import datetime
from typing import Optional, List, Dict, Any
import os
from datetime import datetime
from typing import Any, Dict, List, Optional
from prisma import Prisma
## VARIABLES
DATABASE_URL = "postgresql://postgres:postgres@localhost:5432/litellm"

1
db_scripts/update_unassigned_teams.py
View file

@ -1,4 +1,5 @@
from prisma import Prisma
from litellm._logging import verbose_logger

8
enterprise/enterprise_hooks/banned_keywords.py
View file

@ -8,16 +8,18 @@
from typing import Literal
from fastapi import HTTPException
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails._content_utils import (
is_text_content_call_type,
iter_message_text,
)
from litellm.integrations.custom_logger import CustomLogger
from litellm._logging import verbose_proxy_logger
from fastapi import HTTPException
class _ENTERPRISE_BannedKeywords(CustomLogger):

16
enterprise/enterprise_hooks/blocked_user_list.py
View file

@ -7,15 +7,17 @@
## This accepts a list of user id's for whom calls will be rejected
from typing import Optional, Literal
import litellm
from litellm.proxy.utils import PrismaClient
from litellm.caching.caching import DualCache
from litellm.proxy._types import UserAPIKeyAuth, LiteLLM_EndUserTable
from litellm.integrations.custom_logger import CustomLogger
from litellm._logging import verbose_proxy_logger
from typing import Literal, Optional
from fastapi import HTTPException
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import LiteLLM_EndUserTable, UserAPIKeyAuth
from litellm.proxy.utils import PrismaClient
class _ENTERPRISE_BlockedUserList(CustomLogger):
# Class variables or attributes

1
enterprise/litellm_enterprise/enterprise_callbacks/send_emails/sendgrid_email.py
View file

@ -15,7 +15,6 @@ from litellm.llms.custom_httpx.http_handler import (
from .base_email import BaseEmailLogger
SENDGRID_API_ENDPOINT = "https://api.sendgrid.com/v3/mail/send"

6
enterprise/litellm_enterprise/proxy/auth/custom_sso_handler.py
View file

@ -51,15 +51,15 @@ class EnterpriseCustomSSOHandler:
from fastapi_sso.sso.base import OpenID
from litellm.integrations.custom_sso_handler import CustomSSOLoginHandler
from litellm.proxy.auth.trusted_proxy_utils import (
require_trusted_proxy_request,
)
from litellm.proxy.proxy_server import (
CommonProxyErrors,
general_settings,
premium_user,
user_custom_ui_sso_sign_in_handler,
)
from litellm.proxy.auth.trusted_proxy_utils import (
require_trusted_proxy_request,
)
if premium_user is not True:
raise ValueError(CommonProxyErrors.not_premium_user.value)

3
gateway/main.py
View file

@ -23,13 +23,12 @@ from litellm.proxy.db.db_url_settings import DatabaseURLSettings
DatabaseURLSettings.from_env().apply_to_env()
from litellm.proxy.proxy_server import app
from gateway.routes.allowlist import (
GATEWAY_EXACT_PATHS,
GATEWAY_MOUNT_PATHS,
GATEWAY_PATH_PREFIXES,
)
from litellm.proxy.proxy_server import app
def _is_gateway_route(route) -> bool:

10
litellm-proxy-extras/litellm_proxy_extras/utils.py
View file

@ -10,14 +10,14 @@ from pathlib import Path
from typing import Optional
from litellm_proxy_extras._logging import logger
from litellm_proxy_extras.replica_identity import (
REPLICA_IDENTITY_FULL_ENV_VAR,
apply_replica_identity_full,
)
from litellm_proxy_extras.prisma_toolchain import (
ensure_prisma_toolchain,
prisma_command_timeout,
)
from litellm_proxy_extras.replica_identity import (
REPLICA_IDENTITY_FULL_ENV_VAR,
apply_replica_identity_full,
)
def str_to_bool(value: Optional[str]) -> bool:
@ -414,7 +414,7 @@ class ProxyExtrasDBManager:
def _strip_prisma_query_params(url: str) -> str:
"""Remove Prisma-specific query params (connection_limit, pool_timeout,
schema, etc.) from DATABASE_URL so psycopg can parse it."""
from urllib.parse import urlparse, urlunparse, parse_qsl, urlencode
from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
parsed = urlparse(url)
if not parsed.query:

1
litellm-proxy-extras/tests/test_setup_database_fail_fast.py
View file

@ -9,7 +9,6 @@ import subprocess
from unittest.mock import patch
import pytest
from litellm_proxy_extras.utils import (
ProxyExtrasDBManager,
_max_migration_timestamp,

2
litellm/proxy/guardrails/guardrail_hooks/tealtiger/engine.py
View file

@ -43,7 +43,7 @@ class Decision:
class TealEngine:
def __init__(self, policies: list[dict], mode: PolicyMode = PolicyMode.ENFORCE):
def __init__(self, policies: list[dict], mode: PolicyMode = PolicyMode.ENFORCE) -> None:
self.policies = policies
self.mode = mode
self.patterns = default_patterns()

4
litellm/proxy/guardrails/guardrail_hooks/tealtiger/tealtiger.py
View file

@ -35,8 +35,8 @@ class TealTigerGuardrail(CustomGuardrail):
self,
policies: list | None = None,
policy_mode: str = "ENFORCE",
**kwargs,
):
**kwargs: object,
) -> None:
self.engine: Final = TealEngine(
policies=policies or DEFAULT_POLICIES,
mode=PolicyMode(policy_mode),

3
migrations/run.py
View file

@ -23,10 +23,11 @@ Env vars:
import os
import sys
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
from litellm_proxy_extras._logging import logger
from litellm_proxy_extras.utils import ProxyExtrasDBManager, str_to_bool
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
def main() -> int:
# Assemble DATABASE_URL from the discrete DATABASE_* env vars, matching

430
proxy_log.txt Normal file
View file

File diff suppressed because one or more lines are too long

3
run_proxy.bat Normal file
View file

@ -0,0 +1,3 @@
@echo off
set PYTHONIOENCODING=utf-8
"C:\Users\pauls\AppData\Local\Programs\Python\Python313\Scripts\litellm.exe" --config "C:\Users\pauls\Documents\litellm\config.yaml" --port 4000 --detailed_debug > proxy_log.txt 2>&1

1
scripts/benchmark_chat_completions_perf.py
View file

@ -34,7 +34,6 @@ from typing import Any, Optional
import aiohttp
from aiohttp import web
DEFAULT_MODEL = "perf-test-model"
DEFAULT_API_KEY = "sk-1234"

3
scripts/benchmark_mock.py
View file

@ -3,12 +3,11 @@
import argparse
import asyncio
import time
import statistics
import time
import aiohttp
REQUEST_BODY = {
"model": "db-openai-endpoint",
"messages": [{"role": "user", "content": "hi"}],

13
scripts/benchmark_proxy_vs_provider.py
View file

@ -57,16 +57,17 @@ OUTPUT:
- Run-to-run variance (when using --runs > 1)
"""
import asyncio
import aiohttp
import time
import json
import argparse
import asyncio
import json
import os
from typing import List, Dict, Any, Optional
import sys
import time
from dataclasses import dataclass, field
from statistics import mean, median, stdev
import sys
from typing import Any, Dict, List, Optional
import aiohttp
from aiohttp import TCPConnector

4
scripts/benchmark_streaming_chunk_overhead.py
View file

@ -40,11 +40,13 @@ from litellm.litellm_core_utils.streaming_handler import (
) # noqa: E402
from litellm.types.utils import ( # noqa: E402
Delta,
GenericStreamingChunk as GChunk,
ModelResponseStream,
StreamingChoices,
Usage,
)
from litellm.types.utils import (
GenericStreamingChunk as GChunk,
)
# ---------------------------------------------------------------------------
# Synthetic chunk fixtures

3
scripts/budget_ratchet_check.py
View file

@ -33,11 +33,12 @@ import argparse
import json
import subprocess
import sys
import tomllib
from pathlib import Path
from types import MappingProxyType
from typing import NamedTuple
import tomllib
REPO_ROOT = Path(__file__).resolve().parent.parent
DEFAULT_BASE = "origin/litellm_internal_staging"
DEFAULT_BUDGETS: tuple[str, ...] = (

6
scripts/check_type_discipline.py
View file

@ -101,17 +101,17 @@ Exit code 1 if any violation is found. Stdlib only.
"""
from __future__ import annotations
import ast
import io
import re
import sys
import tokenize
from collections.abc import Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from collections.abc import Iterable, Iterator, Mapping, Sequence
from typing import NamedTuple
# Mutable collection types, banned in *every* annotation. Name-based, so `dict`,
# `typing.Dict`, `collections.deque`, and `collections.abc.MutableMapping` all match
# however they were imported. The read-only interfaces (Mapping, Sequence, the

3
scripts/mutation_report.py
View file

@ -18,12 +18,13 @@ import json
import re
import subprocess
import sys
import tomllib
from collections import defaultdict
from difflib import SequenceMatcher
from pathlib import Path
from textwrap import dedent
import tomllib
ROOT = Path(__file__).resolve().parent.parent
MUTMUT_INVOCATION = ["uv", "run", "--no-sync", "--with", "mutmut==3.5.0", "mutmut"]

12
with_tealtiger.txt Normal file
View file

@ -0,0 +1,12 @@
ImportError while loading conftest 'C:\Users\pauls\Documents\litellm\tests\guardrails_tests\conftest.py'.
tests\guardrails_tests\conftest.py:17: in <module>
import litellm
litellm\__init__.py:44: in <module>
from litellm.types.integrations.datadog import DatadogInitParams
litellm\types\integrations\datadog.py:6: in <module>
from litellm.types.integrations.custom_logger import StandardCustomLoggerInitParams
litellm\types\integrations\custom_logger.py:26: in <module>
class StandardCustomLoggerInitParams(BaseModel):
litellm\types\integrations\custom_logger.py:31: in StandardCustomLoggerInitParams
turn_off_message_logging: bool | None = False
E TypeError: unsupported operand type(s) for |: 'type' and 'NoneType'

3761
without_tealtiger.txt Normal file
View file

File diff suppressed because one or more lines are too long