From 9783a8aa7c7159efa0ad245e8f65de4839088e7c Mon Sep 17 00:00:00 2001 From: joshualipman123 Date: Wed, 30 Jul 2025 10:17:54 -0700 Subject: [PATCH] Update ai gateway docs --- docs/my-website/docs/index.md | 177 +++++++++++++++++- .../docs/providers/vercel_ai_gateway.md | 2 + model_prices_and_context_window.json | 24 +-- 3 files changed, 187 insertions(+), 16 deletions(-) diff --git a/docs/my-website/docs/index.md b/docs/my-website/docs/index.md index 85226f126f4..9055e0cbbef 100644 --- a/docs/my-website/docs/index.md +++ b/docs/my-website/docs/index.md @@ -72,6 +72,158 @@ response = completion( ) ``` + + + +```python +from litellm import completion +import os + +## set ENV variables +os.environ["ANTHROPIC_API_KEY"] = "your-api-key" + +response = completion( + model="anthropic/claude-3-sonnet-20240229", + messages=[{ "content": "Hello, how are you?","role": "user"}] +) +``` + + + + +```python +from litellm import completion +import os + +## set ENV variables +os.environ["XAI_API_KEY"] = "your-api-key" + +response = completion( + model="xai/grok-2-latest", + messages=[{ "content": "Hello, how are you?","role": "user"}] +) +``` + + + +```python +from litellm import completion +import os + +# auth: run 'gcloud auth application-default' +os.environ["VERTEXAI_PROJECT"] = "hardy-device-386718" +os.environ["VERTEXAI_LOCATION"] = "us-central1" + +response = completion( + model="vertex_ai/gemini-1.5-pro", + messages=[{ "content": "Hello, how are you?","role": "user"}] +) +``` + + + + + +```python +from litellm import completion +import os + +## set ENV variables +os.environ["NVIDIA_NIM_API_KEY"] = "nvidia_api_key" +os.environ["NVIDIA_NIM_API_BASE"] = "nvidia_nim_endpoint_url" + +response = completion( + model="nvidia_nim/", + messages=[{ "content": "Hello, how are you?","role": "user"}] +) +``` + + + + + +```python +from litellm import completion +import os + +os.environ["HUGGINGFACE_API_KEY"] = "huggingface_api_key" + +# e.g. Call 'WizardLM/WizardCoder-Python-34B-V1.0' hosted on HF Inference endpoints +response = completion( + model="huggingface/WizardLM/WizardCoder-Python-34B-V1.0", + messages=[{ "content": "Hello, how are you?","role": "user"}], + api_base="https://my-endpoint.huggingface.cloud" +) + +print(response) +``` + + + + + +```python +from litellm import completion +import os + +## set ENV variables +os.environ["AZURE_API_KEY"] = "" +os.environ["AZURE_API_BASE"] = "" +os.environ["AZURE_API_VERSION"] = "" + +# azure call +response = completion( + "azure/", + messages = [{ "content": "Hello, how are you?","role": "user"}] +) +``` + + + + + +```python +from litellm import completion + +response = completion( + model="ollama/llama2", + messages = [{ "content": "Hello, how are you?","role": "user"}], + api_base="http://localhost:11434" +) +``` + + + + +```python +from litellm import completion +import os + +## set ENV variables +os.environ["OPENROUTER_API_KEY"] = "openrouter_api_key" + +response = completion( + model="openrouter/google/palm-2-chat-bison", + messages = [{ "content": "Hello, how are you?","role": "user"}], +) +``` + + + + +```python +from litellm import completion +import os + +## set ENV variables. Visit https://novita.ai/settings/key-management to get your API key +os.environ["NOVITA_API_KEY"] = "novita-api-key" + +response = completion( + model="novita/deepseek/deepseek-r1", + messages=[{ "content": "Hello, how are you?","role": "user"}] +) +``` + @@ -80,13 +232,12 @@ response = completion( from litellm import completion import os -## set ENV variables -os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-api-key" +## set ENV variables. +os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-vercel-api-key" response = completion( model="vercel_ai_gateway/openai/gpt-4o", - messages=[{ "content": "Hello, how are you?","role": "user"}], - stream=True, + messages=[{ "content": "Hello, how are you?","role": "user"}] ) ``` @@ -312,6 +463,24 @@ response = completion( + + +```python +from litellm import completion +import os + +## set ENV variables. V +os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-vercel-api-key" + +response = completion( + model="vercel_ai_gateway/openai/gpt-4o", + messages = [{ "content": "Hello, how are you?","role": "user"}], + stream=True, +) +``` + + + ### Streaming Response Format (OpenAI Format) diff --git a/docs/my-website/docs/providers/vercel_ai_gateway.md b/docs/my-website/docs/providers/vercel_ai_gateway.md index e7b6400d6ab..9a122edfa53 100644 --- a/docs/my-website/docs/providers/vercel_ai_gateway.md +++ b/docs/my-website/docs/providers/vercel_ai_gateway.md @@ -28,6 +28,8 @@ os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "" # your Vercel AI Gateway API key os.environ["VERCEL_OIDC_TOKEN"] = "" # your Vercel OIDC token for authentication ``` +Note: see the [Vercel AI Gateway docs](https://vercel.com/docs/ai-gateway#using-the-ai-gateway-with-an-api-key) for instructions on obtaining a key. + ## Usage - LiteLLM Python SDK ### Non-streaming diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e81c724c3af..dee72d84a60 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -17693,10 +17693,10 @@ "mode": "chat" }, "vercel_ai_gateway/deepseek/deepseek-r1": { - "max_tokens": 64000, + "max_tokens": 128000, "input_cost_per_token": 5.5e-07, "output_cost_per_token": 2.19e-06, - "max_input_tokens": 64000, + "max_input_tokens": 128000, "litellm_provider": "vercel_ai_gateway", "mode": "chat" }, @@ -17999,10 +17999,10 @@ "mode": "chat" }, "vercel_ai_gateway/meta/llama-3.1-8b": { - "max_tokens": 128000, + "max_tokens": 131000, "input_cost_per_token": 5e-08, "output_cost_per_token": 8e-08, - "max_input_tokens": 128000, + "max_input_tokens": 131000, "litellm_provider": "vercel_ai_gateway", "mode": "chat" }, @@ -18426,10 +18426,10 @@ "mode": "chat" }, "vercel_ai_gateway/openai/gpt-3.5-turbo": { - "max_tokens": 4096, + "max_tokens": 16385, "input_cost_per_token": 5e-07, "output_cost_per_token": 1.5e-06, - "max_input_tokens": 4096, + "max_input_tokens": 16385, "litellm_provider": "vercel_ai_gateway", "mode": "chat" }, @@ -19278,10 +19278,10 @@ "mode": "chat" }, "vercel_ai_gateway/mistral/mixtral-8x22b-instruct": { - "max_tokens": 2048, + "max_tokens": 65536, "input_cost_per_token": 1.2e-06, "output_cost_per_token": 1.2e-06, - "max_input_tokens": 2048, + "max_input_tokens": 65536, "litellm_provider": "vercel_ai_gateway", "mode": "chat" }, @@ -19496,9 +19496,9 @@ }, "openrouter/qwen/qwen3-235b-a22b-2507": { "max_tokens": 262144, - "input_cost_per_token": 1.2e-07, + "input_cost_per_token": 1.179e-07, "max_output_tokens": 262144, - "output_cost_per_token": 5.9e-07, + "output_cost_per_token": 1.179e-07, "litellm_provider": "openrouter", "mode": "chat" }, @@ -19743,10 +19743,10 @@ "mode": "chat" }, "vercel_ai_gateway/openai/gpt-3.5-turbo-instruct": { - "max_tokens": 4096, + "max_tokens": 8192, "input_cost_per_token": 1.5e-06, "output_cost_per_token": 2e-06, - "max_input_tokens": 4096, + "max_input_tokens": 8192, "litellm_provider": "vercel_ai_gateway", "mode": "chat" },