diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d429bc6b8c1..a33473b7246 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,12 +1,12 @@ repos: - repo: local hooks: - # - id: mypy - # name: mypy - # entry: python3 -m mypy --ignore-missing-imports - # language: system - # types: [python] - # files: ^litellm/ + - id: mypy + name: mypy + entry: python3 -m mypy --ignore-missing-imports + language: system + types: [python] + files: ^litellm/ - id: isort name: isort entry: isort diff --git a/README.md b/README.md index 306f07ec26f..2153ae948ed 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@

Call all LLM APIs using the OpenAI format [Bedrock, Huggingface, VertexAI, TogetherAI, Azure, OpenAI, Groq etc.]

-

OpenAI Proxy Server | Hosted Proxy (Preview) | Enterprise Tier

+

LiteLLM Proxy Server | Hosted Proxy (Preview) | Enterprise Tier

PyPI Version @@ -35,7 +35,7 @@ LiteLLM manages: - Translate inputs to provider's `completion`, `embedding`, and `image_generation` endpoints - [Consistent output](https://docs.litellm.ai/docs/completion/output), text responses will always be available at `['choices'][0]['message']['content']` - Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing) -- Set Budgets & Rate limits per project, api key, model [OpenAI Proxy Server](https://docs.litellm.ai/docs/simple_proxy) +- Set Budgets & Rate limits per project, api key, model [LiteLLM Proxy Server](https://docs.litellm.ai/docs/simple_proxy) [**Jump to OpenAI Proxy Docs**](https://github.com/BerriAI/litellm?tab=readme-ov-file#openai-proxy---docs)
[**Jump to Supported LLM Providers**](https://github.com/BerriAI/litellm?tab=readme-ov-file#supported-providers-docs) diff --git a/cookbook/litellm_router/error_log.txt b/cookbook/litellm_router/error_log.txt index 6853ef4659a..983b47cbbba 100644 --- a/cookbook/litellm_router/error_log.txt +++ b/cookbook/litellm_router/error_log.txt @@ -1,10 +1,10 @@ -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -21,13 +21,13 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -49,7 +49,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -61,7 +61,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -70,7 +70,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -79,7 +79,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -109,7 +109,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -128,7 +128,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -148,7 +148,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -162,7 +162,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -174,7 +174,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -184,7 +184,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -193,19 +193,19 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -214,7 +214,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -234,7 +234,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -244,7 +244,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -253,7 +253,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -267,31 +267,31 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -305,7 +305,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -330,7 +330,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -339,7 +339,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -360,7 +360,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -369,7 +369,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -378,7 +378,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -388,7 +388,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -409,7 +409,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -422,13 +422,13 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -438,7 +438,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -462,7 +462,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -482,7 +482,7 @@ Exception: 'Response' object has no attribute 'get' Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -492,7 +492,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -516,7 +516,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -529,7 +529,7 @@ Exception: 'Response' object has no attribute 'get' Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -546,13 +546,13 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -580,13 +580,13 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -624,7 +624,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -638,13 +638,13 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -660,7 +660,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -681,7 +681,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -691,31 +691,31 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -771,7 +771,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -780,7 +780,7 @@ Exception: 'Response' object has no attribute 'get' Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -800,7 +800,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -820,7 +820,7 @@ Exception: 'Response' object has no attribute 'get' Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -830,7 +830,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -840,7 +840,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -850,7 +850,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -862,13 +862,13 @@ Exception: 'Response' object has no attribute 'get' Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -877,7 +877,7 @@ Exception: 'Response' object has no attribute 'get' Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -898,7 +898,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -919,7 +919,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -936,19 +936,19 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -961,25 +961,25 @@ Exception: 'Response' object has no attribute 'get' Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -993,7 +993,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 diff --git a/cookbook/litellm_router/request_log.txt b/cookbook/litellm_router/request_log.txt index 0aed7490496..821d87ab56a 100644 --- a/cookbook/litellm_router/request_log.txt +++ b/cookbook/litellm_router/request_log.txt @@ -20,7 +20,7 @@ Call all LLM APIs using the OpenAI format. Response ID: 52dbbd49-eedb-4c11-8382-3ca7deb1af35 Url: /queue/response/52dbbd49-eedb-4c11-8382-3ca7deb1af35 Time: 3.50 seconds -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -35,7 +35,7 @@ Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. C Response ID: ae1e2b71-d711-456d-8df0-13ce0709eb04 Url: /queue/response/ae1e2b71-d711-456d-8df0-13ce0709eb04 Time: 5.60 seconds -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 diff --git a/cookbook/litellm_router/test_questions/question3.txt b/cookbook/litellm_router/test_questions/question3.txt index a122787504e..d6006f9c73c 100644 --- a/cookbook/litellm_router/test_questions/question3.txt +++ b/cookbook/litellm_router/test_questions/question3.txt @@ -1,4 +1,4 @@ -What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 100+ LLMs Huggingface/Bedrock/TogetherAI/etc. in the OpenAI ChatCompletions & Completions format diff --git a/docs/my-website/docs/budget_manager.md b/docs/my-website/docs/budget_manager.md index 1a2c7e7eecd..6bea96ef9ce 100644 --- a/docs/my-website/docs/budget_manager.md +++ b/docs/my-website/docs/budget_manager.md @@ -7,14 +7,14 @@ Don't want to get crazy bills because either while you're calling LLM APIs **or* :::info -If you want a server to manage user keys, budgets, etc. use our [OpenAI Proxy Server](./proxy/virtual_keys.md) +If you want a server to manage user keys, budgets, etc. use our [LiteLLM Proxy Server](./proxy/virtual_keys.md) ::: LiteLLM exposes: * `litellm.max_budget`: a global variable you can use to set the max budget (in USD) across all your litellm calls. If this budget is exceeded, it will raise a BudgetExceededError * `BudgetManager`: A class to help set budgets per user. BudgetManager creates a dictionary to manage the user budgets, where the key is user and the object is their current cost + model-specific costs. -* `OpenAI Proxy Server`: A server to call 100+ LLMs with an openai-compatible endpoint. Manages user budgets, spend tracking, load balancing etc. +* `LiteLLM Proxy Server`: A server to call 100+ LLMs with an openai-compatible endpoint. Manages user budgets, spend tracking, load balancing etc. ## quick start diff --git a/docs/my-website/docs/index.md b/docs/my-website/docs/index.md index 6b472ee6c6d..a560ecf76da 100644 --- a/docs/my-website/docs/index.md +++ b/docs/my-website/docs/index.md @@ -10,11 +10,11 @@ https://github.com/BerriAI/litellm - Translate inputs to provider's `completion`, `embedding`, and `image_generation` endpoints - [Consistent output](https://docs.litellm.ai/docs/completion/output), text responses will always be available at `['choices'][0]['message']['content']` - Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing) -- Track spend & set budgets per project [OpenAI Proxy Server](https://docs.litellm.ai/docs/simple_proxy) +- Track spend & set budgets per project [LiteLLM Proxy Server](https://docs.litellm.ai/docs/simple_proxy) ## How to use LiteLLM You can use litellm through either: -1. [OpenAI proxy Server](#openai-proxy) - Server to call 100+ LLMs, load balance, cost tracking across projects +1. [LiteLLM Proxy Server](#openai-proxy) - Server to call 100+ LLMs, load balance, cost tracking across projects 2. [LiteLLM python SDK](#basic-usage) - Python Client to call 100+ LLMs, load balance, cost tracking ## LiteLLM Python SDK diff --git a/docs/my-website/docs/proxy/custom_pricing.md b/docs/my-website/docs/proxy/custom_pricing.md index 0b747f1193b..51634021b73 100644 --- a/docs/my-website/docs/proxy/custom_pricing.md +++ b/docs/my-website/docs/proxy/custom_pricing.md @@ -1,6 +1,6 @@ import Image from '@theme/IdealImage'; -# Custom Pricing - Sagemaker, etc. +# Custom LLM Pricing - Sagemaker, Azure, etc Use this to register custom pricing for models. @@ -16,39 +16,9 @@ LiteLLM already has pricing for any model in our [model cost map](https://github ::: -## Quick Start +## Cost Per Second (e.g. Sagemaker) -Register custom pricing for sagemaker completion model. - -For cost per second pricing, you **just** need to register `input_cost_per_second`. - -```python -# !pip install boto3 -from litellm import completion, completion_cost - -os.environ["AWS_ACCESS_KEY_ID"] = "" -os.environ["AWS_SECRET_ACCESS_KEY"] = "" -os.environ["AWS_REGION_NAME"] = "" - - -def test_completion_sagemaker(): - try: - print("testing sagemaker") - response = completion( - model="sagemaker/berri-benchmarking-Llama-2-70b-chat-hf-4", - messages=[{"role": "user", "content": "Hey, how's it going?"}], - input_cost_per_second=0.000420, - ) - # Add any assertions here to check the response - print(response) - cost = completion_cost(completion_response=response) - print(cost) - except Exception as e: - raise Exception(f"Error occurred: {e}") - -``` - -### Usage with OpenAI Proxy Server +### Usage with LiteLLM Proxy Server **Step 1: Add pricing to config.yaml** ```yaml @@ -75,38 +45,7 @@ litellm /path/to/config.yaml ## Cost Per Token (e.g. Azure) - -```python -# !pip install boto3 -from litellm import completion, completion_cost - -## set ENV variables -os.environ["AZURE_API_KEY"] = "" -os.environ["AZURE_API_BASE"] = "" -os.environ["AZURE_API_VERSION"] = "" - - -def test_completion_azure_model(): - try: - print("testing azure custom pricing") - # azure call - response = completion( - model = "azure/", - messages = [{ "content": "Hello, how are you?","role": "user"}] - input_cost_per_token=0.005, - output_cost_per_token=1, - ) - # Add any assertions here to check the response - print(response) - cost = completion_cost(completion_response=response) - print(cost) - except Exception as e: - raise Exception(f"Error occurred: {e}") - -test_completion_azure_model() -``` - -### Usage with OpenAI Proxy Server +### Usage with LiteLLM Proxy Server ```yaml model_list: diff --git a/docs/my-website/docs/proxy/deploy.md b/docs/my-website/docs/proxy/deploy.md index 35fc0a50866..c7617196e58 100644 --- a/docs/my-website/docs/proxy/deploy.md +++ b/docs/my-website/docs/proxy/deploy.md @@ -246,7 +246,7 @@ helm install lite-helm ./litellm-helm kubectl --namespace default port-forward $POD_NAME 8080:$CONTAINER_PORT ``` -Your OpenAI proxy server is now running on `http://127.0.0.1:4000`. +Your LiteLLM Proxy Server is now running on `http://127.0.0.1:4000`. @@ -301,7 +301,7 @@ docker run \ --config /app/config.yaml --detailed_debug ``` -Your OpenAI proxy server is now running on `http://0.0.0.0:4000`. +Your LiteLLM Proxy Server is now running on `http://0.0.0.0:4000`. @@ -399,7 +399,7 @@ kubectl apply -f /path/to/service.yaml kubectl port-forward service/litellm-service 4000:4000 ``` -Your OpenAI proxy server is now running on `http://0.0.0.0:4000`. +Your LiteLLM Proxy Server is now running on `http://0.0.0.0:4000`. @@ -441,7 +441,7 @@ kubectl \ 4000:4000 ``` -Your OpenAI proxy server is now running on `http://127.0.0.1:4000`. +Your LiteLLM Proxy Server is now running on `http://127.0.0.1:4000`. If you need to set your litellm proxy config.yaml, you can find this in [values.yaml](https://github.com/BerriAI/litellm/blob/main/deploy/charts/litellm-helm/values.yaml) @@ -486,7 +486,7 @@ helm install lite-helm ./litellm-helm kubectl --namespace default port-forward $POD_NAME 8080:$CONTAINER_PORT ``` -Your OpenAI proxy server is now running on `http://127.0.0.1:4000`. +Your LiteLLM Proxy Server is now running on `http://127.0.0.1:4000`. diff --git a/docs/my-website/docs/proxy_server.md b/docs/my-website/docs/proxy_server.md index ef9352ab1f7..0d08db7444e 100644 --- a/docs/my-website/docs/proxy_server.md +++ b/docs/my-website/docs/proxy_server.md @@ -1,7 +1,7 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# [OLD PROXY 👉 [NEW proxy here](./simple_proxy)] Local OpenAI Proxy Server +# [OLD PROXY 👉 [NEW proxy here](./simple_proxy)] Local LiteLLM Proxy Server A fast, and lightweight OpenAI-compatible server to call 100+ LLM APIs. diff --git a/docs/my-website/docs/routing.md b/docs/my-website/docs/routing.md index 905954e9797..d83755e68dd 100644 --- a/docs/my-website/docs/routing.md +++ b/docs/my-website/docs/routing.md @@ -14,7 +14,7 @@ In production, litellm supports using Redis as a way to track cooldown server an :::info -If you want a server to load balance across different LLM APIs, use our [OpenAI Proxy Server](./proxy/load_balancing.md) +If you want a server to load balance across different LLM APIs, use our [LiteLLM Proxy Server](./proxy/load_balancing.md) ::: @@ -1637,7 +1637,7 @@ response = router.completion( ## Deploy Router -If you want a server to load balance across different LLM APIs, use our [OpenAI Proxy Server](./simple_proxy#load-balancing---multiple-instances-of-1-model) +If you want a server to load balance across different LLM APIs, use our [LiteLLM Proxy Server](./simple_proxy#load-balancing---multiple-instances-of-1-model) ## Init Params for the litellm.Router diff --git a/docs/my-website/docs/sdk_custom_pricing.md b/docs/my-website/docs/sdk_custom_pricing.md new file mode 100644 index 00000000000..c8577115109 --- /dev/null +++ b/docs/my-website/docs/sdk_custom_pricing.md @@ -0,0 +1,65 @@ +# Custom Pricing - SageMaker, Azure, etc + +Register custom pricing for sagemaker completion model. + +For cost per second pricing, you **just** need to register `input_cost_per_second`. + +```python +# !pip install boto3 +from litellm import completion, completion_cost + +os.environ["AWS_ACCESS_KEY_ID"] = "" +os.environ["AWS_SECRET_ACCESS_KEY"] = "" +os.environ["AWS_REGION_NAME"] = "" + + +def test_completion_sagemaker(): + try: + print("testing sagemaker") + response = completion( + model="sagemaker/berri-benchmarking-Llama-2-70b-chat-hf-4", + messages=[{"role": "user", "content": "Hey, how's it going?"}], + input_cost_per_second=0.000420, + ) + # Add any assertions here to check the response + print(response) + cost = completion_cost(completion_response=response) + print(cost) + except Exception as e: + raise Exception(f"Error occurred: {e}") + +``` + + +## Cost Per Token (e.g. Azure) + + +```python +# !pip install boto3 +from litellm import completion, completion_cost + +## set ENV variables +os.environ["AZURE_API_KEY"] = "" +os.environ["AZURE_API_BASE"] = "" +os.environ["AZURE_API_VERSION"] = "" + + +def test_completion_azure_model(): + try: + print("testing azure custom pricing") + # azure call + response = completion( + model = "azure/", + messages = [{ "content": "Hello, how are you?","role": "user"}] + input_cost_per_token=0.005, + output_cost_per_token=1, + ) + # Add any assertions here to check the response + print(response) + cost = completion_cost(completion_response=response) + print(cost) + except Exception as e: + raise Exception(f"Error occurred: {e}") + +test_completion_azure_model() +``` \ No newline at end of file diff --git a/docs/my-website/docs/secret.md b/docs/my-website/docs/secret.md index 91ae383686e..c2b6774c0b1 100644 --- a/docs/my-website/docs/secret.md +++ b/docs/my-website/docs/secret.md @@ -61,7 +61,7 @@ litellm --config /path/to/config.yaml ``` ## Azure Key Vault - + -### Usage with OpenAI Proxy Server +### Usage with LiteLLM Proxy Server 1. Install Proxy dependencies ```bash @@ -129,7 +129,7 @@ litellm --config /path/to/config.yaml Use encrypted keys from Google KMS on the proxy -### Usage with OpenAI Proxy Server +### Usage with LiteLLM Proxy Server ## Step 1. Add keys to env ``` @@ -160,29 +160,6 @@ $ litellm --test [Quick Test Proxy](./proxy/quick_start#using-litellm-proxy---curl-request-openai-package-langchain-langchain-js) - -## Infisical Secret Manager -Integrates with [Infisical's Secret Manager](https://infisical.com/) for secure storage and retrieval of API keys and sensitive data. - -### Usage -liteLLM manages reading in your LLM API secrets/env variables from Infisical for you - -```python -import litellm -from infisical import InfisicalClient - -litellm.secret_manager = InfisicalClient(token="your-token") - -messages = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What's the weather like today?"}, -] - -response = litellm.completion(model="gpt-3.5-turbo", messages=messages) - -print(response) -``` - - + diff --git a/docs/my-website/docs/simple_proxy_old_doc.md b/docs/my-website/docs/simple_proxy_old_doc.md index 195728d1be4..2d68db32964 100644 --- a/docs/my-website/docs/simple_proxy_old_doc.md +++ b/docs/my-website/docs/simple_proxy_old_doc.md @@ -2,7 +2,7 @@ import Image from '@theme/IdealImage'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# 💥 OpenAI Proxy Server +# 💥 LiteLLM Proxy Server LiteLLM Server manages: diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index a57a9aecd7d..65483e3925a 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -20,10 +20,10 @@ const sidebars = { { type: "doc", id: "index" }, // NEW { type: "category", - label: "💥 OpenAI Proxy Server", + label: "💥 LiteLLM Proxy Server", link: { type: "generated-index", - title: "💥 OpenAI Proxy Server", + title: "💥 LiteLLM Proxy Server", description: `Proxy Server to call 100+ LLMs in a unified interface & track spend, set budgets per virtual key/user`, slug: "/simple_proxy", }, @@ -42,6 +42,7 @@ const sidebars = { "proxy/configs", "proxy/reliability", "proxy/cost_tracking", + "proxy/custom_pricing", "proxy/self_serve", "proxy/virtual_keys", { @@ -49,6 +50,14 @@ const sidebars = { label: "🪢 Logging", items: ["proxy/logging", "proxy/bucket", "proxy/streaming_logging"], }, + { + type: "category", + label: "Secret Manager - storing LLM API Keys", + items: [ + "secret", + "oidc" + ] + }, "proxy/team_logging", "proxy/guardrails", "proxy/tag_routing", @@ -127,7 +136,7 @@ const sidebars = { }, { type: "category", - label: "Supported Models & Providers", + label: "💯 Supported Models & Providers", link: { type: "generated-index", title: "Providers", @@ -184,20 +193,67 @@ const sidebars = { ], }, - "proxy/custom_pricing", - "routing", - "scheduler", - "set_keys", - "budget_manager", { - type: "category", - label: "Secret Manager", + type: "category", + label: "litellm.completion()", + link: { + type: "generated-index", + title: "Completion()", + description: "Details on the completion() function", + slug: "/completion", + }, items: [ - "secret", - "oidc" - ] + "completion/input", + "completion/provider_specific_params", + "completion/json_mode", + "completion/drop_params", + "completion/prompt_formatting", + "completion/output", + "exception_mapping", + "completion/stream", + "completion/message_trimming", + "completion/function_call", + "completion/vision", + "completion/model_alias", + "completion/batching", + "completion/mock_requests", + "completion/reliable_completions", + ], + }, + { + type: "category", + label: "Embedding(), Image Generation(), Assistants(), Moderation(), Audio Transcriptions(), TTS(), Batches(), Fine-Tuning()", + items: [ + "embedding/supported_embedding", + "embedding/async_embedding", + "embedding/moderation", + "image_generation", + "audio_transcription", + "text_to_speech", + "assistants", + "batches", + "fine_tuning", + "anthropic_completion" + ], + }, + { + type: "category", + label: "🚅 LiteLLM Python SDK", + items: [ + "routing", + "scheduler", + "set_keys", + "completion/token_usage", + "sdk_custom_pricing", + "budget_manager", + "caching/all_caches", + { + type: "category", + label: "LangChain, LlamaIndex, Instructor Integration", + items: ["langchain/langchain", "tutorials/instructor"], + }, + ], }, - "completion/token_usage", "load_test", { type: "category", @@ -228,14 +284,12 @@ const sidebars = { `observability/telemetry`, ], }, - "caching/all_caches", { type: "category", label: "Tutorials", items: [ 'tutorials/azure_openai', 'tutorials/instructor', - 'tutorials/oobabooga', "tutorials/gradio_integration", "tutorials/huggingface_codellama", "tutorials/huggingface_tutorial", @@ -247,11 +301,6 @@ const sidebars = { "tutorials/model_fallbacks", ], }, - { - type: "category", - label: "LangChain, LlamaIndex, Instructor Integration", - items: ["langchain/langchain", "tutorials/instructor"], - }, { type: "category", label: "Extras", diff --git a/docs/my-website/src/pages/index.md b/docs/my-website/src/pages/index.md index 308ed083175..36d47aedf7c 100644 --- a/docs/my-website/src/pages/index.md +++ b/docs/my-website/src/pages/index.md @@ -10,7 +10,7 @@ https://github.com/BerriAI/litellm - Translate inputs to provider's `completion`, `embedding`, and `image_generation` endpoints - [Consistent output](https://docs.litellm.ai/docs/completion/output), text responses will always be available at `['choices'][0]['message']['content']` - Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing) -- Track spend & set budgets per project [OpenAI Proxy Server](https://docs.litellm.ai/docs/simple_proxy) +- Track spend & set budgets per project [LiteLLM Proxy Server](https://docs.litellm.ai/docs/simple_proxy) ## Basic usage diff --git a/litellm/llms/bedrock.py b/litellm/llms/bedrock.py index e12b656ed1f..2185ec459e3 100644 --- a/litellm/llms/bedrock.py +++ b/litellm/llms/bedrock.py @@ -13,6 +13,7 @@ from enum import Enum from typing import Any, Callable, List, Optional, Union import httpx +from openai.types.image import Image import litellm from litellm.litellm_core_utils.core_helpers import map_finish_reason @@ -1413,10 +1414,10 @@ def embedding( def image_generation( model: str, prompt: str, + model_response: ImageResponse, + optional_params: dict, timeout=None, logging_obj=None, - model_response=None, - optional_params=None, aimg_generation=False, ): """ @@ -1513,9 +1514,10 @@ def image_generation( if model_response is None: model_response = ImageResponse() - image_list: List = [] + image_list: List[Image] = [] for artifact in response_body["artifacts"]: - image_dict = {"url": artifact["base64"]} + _image = Image(b64_json=artifact["base64"]) + image_list.append(_image) - model_response.data = image_dict + model_response.data = image_list return model_response diff --git a/litellm/llms/vertex_httpx.py b/litellm/llms/vertex_httpx.py index 9995373f32e..954a30b8010 100644 --- a/litellm/llms/vertex_httpx.py +++ b/litellm/llms/vertex_httpx.py @@ -13,6 +13,7 @@ from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union import httpx # type: ignore import requests # type: ignore +from openai.types.image import Image import litellm import litellm.litellm_core_utils @@ -1341,10 +1342,10 @@ class VertexLLM(BaseLLM): _json_response = response.json() _predictions = _json_response["predictions"] - _response_data: List[litellm.ImageObject] = [] + _response_data: List[Image] = [] for _prediction in _predictions: _bytes_base64_encoded = _prediction["bytesBase64Encoded"] - image_object = litellm.ImageObject(b64_json=_bytes_base64_encoded) + image_object = Image(b64_json=_bytes_base64_encoded) _response_data.append(image_object) model_response.data = _response_data @@ -1453,10 +1454,10 @@ class VertexLLM(BaseLLM): _json_response = response.json() _predictions = _json_response["predictions"] - _response_data: List[litellm.ImageObject] = [] + _response_data: List[Image] = [] for _prediction in _predictions: _bytes_base64_encoded = _prediction["bytesBase64Encoded"] - image_object = litellm.ImageObject(b64_json=_bytes_base64_encoded) + image_object = Image(b64_json=_bytes_base64_encoded) _response_data.append(image_object) model_response.data = _response_data diff --git a/litellm/tests/test_image_generation.py b/litellm/tests/test_image_generation.py index 71f2e761efa..e59cf286519 100644 --- a/litellm/tests/test_image_generation.py +++ b/litellm/tests/test_image_generation.py @@ -158,7 +158,11 @@ def test_image_generation_bedrock(): model="bedrock/stability.stable-diffusion-xl-v1", aws_region_name="us-west-2", ) + print(f"response: {response}") + from openai.types.images_response import ImagesResponse + + ImagesResponse.model_validate(response.model_dump()) except litellm.RateLimitError as e: pass except litellm.ContentPolicyViolationError: diff --git a/litellm/tests/test_optional_params.py b/litellm/tests/test_optional_params.py index 83ac855a813..b2b0a0a2a4a 100644 --- a/litellm/tests/test_optional_params.py +++ b/litellm/tests/test_optional_params.py @@ -420,3 +420,21 @@ def test_dynamic_drop_additional_params_e2e(): print(mock_response.call_args.kwargs["data"]) assert "response_format" not in mock_response.call_args.kwargs["data"] assert "additional_drop_params" not in mock_response.call_args.kwargs["data"] + + +def test_get_optional_params_image_gen(): + response = litellm.utils.get_optional_params_image_gen( + aws_region_name="us-east-1", custom_llm_provider="openai" + ) + + print(response) + + assert "aws_region_name" not in response + + response = litellm.utils.get_optional_params_image_gen( + aws_region_name="us-east-1", custom_llm_provider="bedrock" + ) + + print(response) + + assert "aws_region_name" in response diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 314370f0233..be77821adb0 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -950,16 +950,18 @@ class ImageObject(OpenAIObject): return self.dict() -class ImageResponse(OpenAIObject): - created: Optional[int] = None +from openai.types.images_response import ImagesResponse as OpenAIImageResponse - data: Optional[List[ImageObject]] = None - - usage: Optional[dict] = None +class ImageResponse(OpenAIImageResponse): _hidden_params: dict = {} - def __init__(self, created=None, data=None, response_ms=None): + def __init__( + self, + created: Optional[int] = None, + data: Optional[list] = None, + response_ms=None, + ): if response_ms: _response_ms = response_ms else: @@ -967,14 +969,14 @@ class ImageResponse(OpenAIObject): if data: data = data else: - data = None + data = [] if created: created = created else: - created = None + created = int(time.time()) - super().__init__(data=data, created=created) + super().__init__(created=created, data=data) self.usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} def __contains__(self, key): diff --git a/litellm/utils.py b/litellm/utils.py index b6d00536f4d..825caf326d6 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2391,6 +2391,18 @@ def get_optional_params_image_gen( additional_drop_params = passed_params.pop("additional_drop_params", None) special_params = passed_params.pop("kwargs") for k, v in special_params.items(): + if k.startswith("aws_") and ( + custom_llm_provider != "bedrock" and custom_llm_provider != "sagemaker" + ): # allow dynamically setting boto3 init logic + continue + elif k == "hf_model_name" and custom_llm_provider != "sagemaker": + continue + elif ( + k.startswith("vertex_") + and custom_llm_provider != "vertex_ai" + and custom_llm_provider != "vertex_ai_beta" + ): # allow dynamically setting vertex ai init logic + continue passed_params[k] = v default_params = {