From 2edcc51e575b97e8740b4ced9b82ae07e6d9d55d Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Tue, 19 Aug 2025 16:52:46 -0400 Subject: [PATCH 1/3] updated claude-code docs and deployment faq --- docs/my-website/docs/proxy/deploy.md | 14 +++ .../docs/tutorials/claude_responses_api.md | 108 ++++++++++-------- 2 files changed, 72 insertions(+), 50 deletions(-) diff --git a/docs/my-website/docs/proxy/deploy.md b/docs/my-website/docs/proxy/deploy.md index 701fc492232..73264a026d4 100644 --- a/docs/my-website/docs/proxy/deploy.md +++ b/docs/my-website/docs/proxy/deploy.md @@ -1010,5 +1010,19 @@ User-agent: * Disallow: / ``` +## Deployment FAQ + +### Database Support + +**Q: Is Postgres the only supported database, or do you support other ones (like Mongo)?** + +A: We explored MySQL but that was hard to maintain and led to bugs for customers. Currently, PostgreSQL is our primary supported database for production deployments. + +### Database Downtime Handling + +**Q: If there is Postgres downtime, how does LiteLLM react? Does it fail-open or is there API downtime?** + +A: You can gracefully handle DB unavailability if it's on your VPC. See our production guide for more details: [Gracefully Handle DB Unavailability](https://docs.litellm.ai/docs/proxy/prod#6-if-running-litellm-on-vpc-gracefully-handle-db-unavailability) + diff --git a/docs/my-website/docs/tutorials/claude_responses_api.md b/docs/my-website/docs/tutorials/claude_responses_api.md index a06be87409b..9917a1fa087 100644 --- a/docs/my-website/docs/tutorials/claude_responses_api.md +++ b/docs/my-website/docs/tutorials/claude_responses_api.md @@ -4,11 +4,11 @@ import TabItem from '@theme/TabItem'; # Claude Code -This tutorial shows how to call the Responses API models like `codex-mini` and `o3-pro` from the Claude Code endpoint on LiteLLM. +This tutorial shows how to call Claude models through LiteLLM proxy from Claude Code. :::info -This tutorial is based on [Anthropic's official LiteLLM configuration documentation](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration). This integration allows you to use any LiteLLM supported model through Claude Code. +This tutorial is based on [Anthropic's official LiteLLM configuration documentation](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration). This integration allows you to use any LiteLLM supported model through Claude Code with centralized authentication, usage tracking, and cost controls. ::: @@ -31,19 +31,18 @@ Create a secure configuration using environment variables: ```yaml model_list: - # Responses API models - - model_name: codex-mini + # Claude models + - model_name: claude-3-5-sonnet-20241022 litellm_params: - model: openai/codex-mini - api_key: os.environ/OPENAI_API_KEY - api_base: https://api.openai.com/v1 + model: anthropic/claude-3-5-sonnet-20241022 + api_key: os.environ/ANTHROPIC_API_KEY - - model_name: o3-pro + - model_name: claude-3-5-haiku-20241022 litellm_params: - model: openai/o3-pro - api_key: os.environ/OPENAI_API_KEY - api_base: https://api.openai.com/v1 + model: anthropic/claude-3-5-haiku-20241022 + api_key: os.environ/ANTHROPIC_API_KEY + litellm_settings: master_key: os.environ/LITELLM_MASTER_KEY ``` @@ -51,7 +50,7 @@ litellm_settings: Set your environment variables: ```bash -export OPENAI_API_KEY="your-openai-api-key" +export ANTHROPIC_API_KEY="your-anthropic-api-key" export LITELLM_MASTER_KEY="sk-1234567890" # Generate a secure key ``` @@ -72,31 +71,43 @@ curl -X POST http://0.0.0.0:4000/v1/messages \ -H "Authorization: Bearer $LITELLM_MASTER_KEY" \ -H "Content-Type: application/json" \ -d '{ - "model": "codex-mini", + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 1000, "messages": [{"role": "user", "content": "What is the capital of France?"}] }' ``` ### 4. Configure Claude Code -Setup Claude Code to use your LiteLLM proxy: +#### Method 1: Unified Endpoint (Recommended) + +Configure Claude Code to use LiteLLM's unified endpoint: ```bash export ANTHROPIC_BASE_URL="http://0.0.0.0:4000" export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY" ``` -### 5. Use Claude Code +#### Method 2: Provider-specific Pass-through Endpoint -Start Claude Code with any configured model: +Alternatively, use the Anthropic pass-through endpoint: ```bash -# Use Responses API models -claude --model codex-mini -claude --model o3-pro +export ANTHROPIC_BASE_URL="http://0.0.0.0:4000/anthropic" +export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY" +``` -# Or use the latest model alias -claude --model codex-mini-latest +### 5. Use Claude Code + +Start Claude Code and it will automatically use your configured models: + +```bash +# Claude Code will use the models configured in your LiteLLM proxy +claude + +# Or specify a model if you have multiple configured +claude --model claude-3-5-sonnet-20241022 +claude --model claude-3-5-haiku-20241022 ``` Example conversation: @@ -112,7 +123,8 @@ Common issues and solutions: **Authentication errors:** - Verify your environment variables are set: `echo $LITELLM_MASTER_KEY` -- Check that your OpenAI API key is valid and has sufficient credits +- Check that your API keys are valid and have sufficient credits +- Ensure the `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key **Model not found:** - Ensure the model name in Claude Code matches exactly with your `config.yaml` @@ -123,48 +135,44 @@ Common issues and solutions: Expand your configuration to support multiple providers and models: - + ```yaml model_list: - # Responses API models - - model_name: codex-mini - litellm_params: - model: openai/codex-mini - api_key: os.environ/OPENAI_API_KEY - api_base: https://api.openai.com/v1 - - - model_name: o3-pro - litellm_params: - model: openai/o3-pro - api_key: os.environ/OPENAI_API_KEY - api_base: https://api.openai.com/v1 - - # Standard models - - model_name: gpt-4o - litellm_params: - model: openai/gpt-4o - api_key: os.environ/OPENAI_API_KEY - - - model_name: claude-3-5-sonnet + # Anthropic models + - model_name: claude-3-5-sonnet-20241022 litellm_params: model: anthropic/claude-3-5-sonnet-20241022 api_key: os.environ/ANTHROPIC_API_KEY + + - model_name: claude-3-5-haiku-20241022 + litellm_params: + model: anthropic/claude-3-5-haiku-20241022 + api_key: os.environ/ANTHROPIC_API_KEY + + # AWS Bedrock + - model_name: claude-bedrock + litellm_params: + model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0 + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + aws_region_name: us-east-1 litellm_settings: master_key: os.environ/LITELLM_MASTER_KEY ``` -Switch between models seamlessly: +Use different models based on your needs: ```bash -# Use Responses API models for advanced reasoning -claude --model o3-pro -claude --model codex-mini +# Use Claude for complex reasoning +claude --model claude-3-5-sonnet-20241022 -# Use standard models for general tasks -claude --model gpt-4o -claude --model claude-3-5-sonnet +# Use Haiku for fast responses +claude --model claude-3-5-haiku-20241022 + +# Use Bedrock deployment +claude --model claude-bedrock ``` From ef9e50458d58569bfdbb00acca416cdf1031bb1b Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Tue, 19 Aug 2025 17:01:31 -0400 Subject: [PATCH 2/3] removed faq --- docs/my-website/docs/proxy/deploy.md | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/docs/my-website/docs/proxy/deploy.md b/docs/my-website/docs/proxy/deploy.md index 73264a026d4..701fc492232 100644 --- a/docs/my-website/docs/proxy/deploy.md +++ b/docs/my-website/docs/proxy/deploy.md @@ -1010,19 +1010,5 @@ User-agent: * Disallow: / ``` -## Deployment FAQ - -### Database Support - -**Q: Is Postgres the only supported database, or do you support other ones (like Mongo)?** - -A: We explored MySQL but that was hard to maintain and led to bugs for customers. Currently, PostgreSQL is our primary supported database for production deployments. - -### Database Downtime Handling - -**Q: If there is Postgres downtime, how does LiteLLM react? Does it fail-open or is there API downtime?** - -A: You can gracefully handle DB unavailability if it's on your VPC. See our production guide for more details: [Gracefully Handle DB Unavailability](https://docs.litellm.ai/docs/proxy/prod#6-if-running-litellm-on-vpc-gracefully-handle-db-unavailability) - From d664860aa15d1c20e42253495b9b66bde7d6d725 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Sat, 23 Aug 2025 10:02:49 -0400 Subject: [PATCH 3/3] added openai models to config --- .../docs/tutorials/claude_responses_api.md | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/docs/my-website/docs/tutorials/claude_responses_api.md b/docs/my-website/docs/tutorials/claude_responses_api.md index 9917a1fa087..09b352a7663 100644 --- a/docs/my-website/docs/tutorials/claude_responses_api.md +++ b/docs/my-website/docs/tutorials/claude_responses_api.md @@ -139,6 +139,25 @@ Expand your configuration to support multiple providers and models: ```yaml model_list: + # OpenAI models + - model_name: codex-mini + litellm_params: + model: openai/codex-mini + api_key: os.environ/OPENAI_API_KEY + api_base: https://api.openai.com/v1 + + - model_name: o3-pro + litellm_params: + model: openai/o3-pro + api_key: os.environ/OPENAI_API_KEY + api_base: https://api.openai.com/v1 + + - model_name: gpt-4o + litellm_params: + model: openai/gpt-4o + api_key: os.environ/OPENAI_API_KEY + api_base: https://api.openai.com/v1 + # Anthropic models - model_name: claude-3-5-sonnet-20241022 litellm_params: @@ -162,7 +181,7 @@ litellm_settings: master_key: os.environ/LITELLM_MASTER_KEY ``` -Use different models based on your needs: +Switch between models seamlessly: ```bash # Use Claude for complex reasoning