@@ -28,10 +28,14 @@ LiteLLM manages:
- Translate inputs to provider's `completion`, `embedding`, and `image_generation` endpoints
- [Consistent output](https://docs.litellm.ai/docs/completion/output), text responses will always be available at `['choices'][0]['message']['content']`
- Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing)
+- Track spend & set budgets per project [OpenAI Proxy Server](https://docs.litellm.ai/docs/simple_proxy)
+
[**Jump to OpenAI Proxy Docs**](https://github.com/BerriAI/litellm?tab=readme-ov-file#openai-proxy---docs) [**Jump to Supported LLM Providers**](https://github.com/BerriAI/litellm?tab=readme-ov-file#supported-provider-docs) +Support for more providers. Missing a provider or LLM Platform, raise a [feature request](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.yml&title=%5BFeature%5D%3A+). + # Usage ([**Docs**](https://docs.litellm.ai/docs/)) > [!IMPORTANT] > LiteLLM v1.0.0 now requires `openai>=1.0.0`. Migration guide [here](https://docs.litellm.ai/docs/migration) @@ -155,6 +159,9 @@ print(response) ``` ## Proxy Key Management ([Docs](https://docs.litellm.ai/docs/proxy/virtual_keys)) +UI on `/ui` on your proxy server + + Track Spend, Set budgets and create virtual keys for the proxy `POST /key/generate` @@ -174,17 +181,6 @@ curl 'http://0.0.0.0:8000/key/generate' \ } ``` -### [Beta] Proxy UI - -A simple UI to add new models and let your users create keys. - -Live here: https://dashboard.litellm.ai/ - -Code: https://github.com/BerriAI/litellm/tree/main/ui - - -
@@ -157,9 +161,6 @@ response = completion(
messages=[{ "content": "Hello, how are you?","role": "user"}],
stream=True,
)
-
-for chunk in response:
- print(chunk)
```
@@ -20,6 +20,27 @@ litellm.vertex_location = "us-central1" # proj location
response = litellm.completion(model="gemini-pro", messages=[{"role": "user", "content": "write code for saying hi from LiteLLM"}])
```
+## OpenAI Proxy Usage
+
+1. Modify the config.yaml
+
+```yaml
+litellm_settings:
+ vertex_project: "hardy-device-38811" # Your Project ID
+ vertex_location: "us-central1" # proj location
+
+model_list:
+ -model_name: team1-gemini-pro
+ litellm_params:
+ model: gemini-pro
+```
+
+2. Start the proxy
+
+```bash
+$ litellm --config /path/to/config.yaml
+```
+
## Set Vertex Project & Vertex Location
All calls using Vertex AI require the following parameters:
* Your Project ID
@@ -46,16 +67,39 @@ os.environ["VERTEXAI_LOCATION"] = "us-central1 # Your Location
# set directly on module
litellm.vertex_location = "us-central1 # Your Location
```
+## Model Garden
+| Model Name | Function Call |
+|------------------|--------------------------------------|
+| llama2 | `completion('vertex_ai/
@@ -306,30 +311,7 @@ litellm.success_callback = ["langfuse", "llmonitor"] # log input/output to langf
response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}])
```
-## Calculate Costs, Usage, Latency
-
-Pass the completion response to `litellm.completion_cost(completion_response=response)` and get the cost
-
-```python
-from litellm import completion, completion_cost
-import os
-os.environ["OPENAI_API_KEY"] = "your-api-key"
-
-response = completion(
- model="gpt-3.5-turbo",
- messages=[{ "content": "Hello, how are you?","role": "user"}]
-)
-
-cost = completion_cost(completion_response=response)
-print("Cost for completion call with gpt-3.5-turbo: ", f"${float(cost):.10f}")
-```
-
-**Output**
-```shell
-Cost for completion call with gpt-3.5-turbo: $0.0000775000
-```
-
-### Track Costs, Usage, Latency for streaming
+## Track Costs, Usage, Latency for streaming
Use a callback function for this - more info on custom callbacks: https://docs.litellm.ai/docs/observability/custom_callback
```python
@@ -342,18 +324,8 @@ def track_cost_callback(
start_time, end_time # start/end time
):
try:
- # check if it has collected an entire stream response
- if "complete_streaming_response" in kwargs:
- # for tracking streaming cost we pass the "messages" and the output_text to litellm.completion_cost
- completion_response=kwargs["complete_streaming_response"]
- input_text = kwargs["messages"]
- output_text = completion_response["choices"][0]["message"]["content"]
- response_cost = litellm.completion_cost(
- model = kwargs["model"],
- messages = input_text,
- completion=output_text
- )
- print("streaming response_cost", response_cost)
+ response_cost = kwargs.get("response_cost", 0)
+ print("streaming response_cost", response_cost)
except:
pass
# set callback
@@ -372,13 +344,12 @@ response = completion(
)
```
-
-Need a dedicated key? Email us @ krrish@berri.ai
-
## OpenAI Proxy
Track spend across multiple projects/people
+
+
The proxy provides:
1. [Hooks for auth](https://docs.litellm.ai/docs/proxy/virtual_keys#custom-auth)
2. [Hooks for logging](https://docs.litellm.ai/docs/proxy/logging#step-1---create-your-custom-litellm-callback-class)
@@ -418,4 +389,4 @@ print(response)
## More details
* [exception mapping](./exception_mapping.md)
* [retries + model fallbacks for completion()](./completion/reliable_completions.md)
-* [tutorial for model fallbacks with completion()](./tutorials/fallbacks.md)
\ No newline at end of file
+* [proxy virtual keys & spend management](./tutorials/fallbacks.md)
\ No newline at end of file
diff --git a/litellm/__init__.py b/litellm/__init__.py
index d67ecb718ad..6a0cb95ae63 100644
--- a/litellm/__init__.py
+++ b/litellm/__init__.py
@@ -1,11 +1,13 @@
### INIT VARIABLES ###
-import threading, requests
+import threading, requests, os
from typing import Callable, List, Optional, Dict, Union, Any
from litellm.caching import Cache
-from litellm._logging import set_verbose, _turn_on_debug
+from litellm._logging import set_verbose, _turn_on_debug, verbose_logger
from litellm.proxy._types import KeyManagementSystem
import httpx
+import dotenv
+dotenv.load_dotenv()
#############################################
if set_verbose == True:
_turn_on_debug()
@@ -62,6 +64,9 @@ cache: Optional[
model_alias_map: Dict[str, str] = {}
model_group_alias_map: Dict[str, str] = {}
max_budget: float = 0.0 # set the max budget across all providers
+budget_duration: Optional[
+ str
+] = None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
_openai_completion_params = [
"functions",
"function_call",
@@ -140,6 +145,10 @@ model_cost_map_url: str = "https://raw.githubusercontent.com/BerriAI/litellm/mai
suppress_debug_info = False
dynamodb_table_name: Optional[str] = None
s3_callback_params: Optional[Dict] = None
+default_key_generate_params: Optional[Dict] = None
+upperbound_key_generate_params: Optional[Dict] = None
+default_team_settings: Optional[List] = None
+max_user_budget: Optional[float] = None
#### RELIABILITY ####
request_timeout: Optional[float] = 6000
num_retries: Optional[int] = None # per model endpoint
@@ -159,6 +168,19 @@ _key_management_system: Optional[KeyManagementSystem] = None
def get_model_cost_map(url: str):
+ if (
+ os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", False) == True
+ or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", False) == "True"
+ ):
+ import importlib.resources
+ import json
+
+ with importlib.resources.open_text(
+ "litellm", "model_prices_and_context_window_backup.json"
+ ) as f:
+ content = json.load(f)
+ return content
+
try:
with requests.get(
url, timeout=5
@@ -214,6 +236,7 @@ vertex_chat_models: List = []
vertex_code_chat_models: List = []
vertex_text_models: List = []
vertex_code_text_models: List = []
+vertex_embedding_models: List = []
ai21_models: List = []
nlp_cloud_models: List = []
aleph_alpha_models: List = []
@@ -243,6 +266,8 @@ for key, value in model_cost.items():
vertex_chat_models.append(key)
elif value.get("litellm_provider") == "vertex_ai-code-chat-models":
vertex_code_chat_models.append(key)
+ elif value.get("litellm_provider") == "vertex_ai-embedding-models":
+ vertex_embedding_models.append(key)
elif value.get("litellm_provider") == "ai21":
ai21_models.append(key)
elif value.get("litellm_provider") == "nlp_cloud":
@@ -262,6 +287,7 @@ openai_compatible_endpoints: List = [
"api.endpoints.anyscale.com/v1",
"api.deepinfra.com/v1/openai",
"api.mistral.ai/v1",
+ "api.together.xyz/v1",
]
# this is maintained for Exception Mapping
@@ -271,6 +297,7 @@ openai_compatible_providers: List = [
"deepinfra",
"perplexity",
"xinference",
+ "together_ai",
]
@@ -479,7 +506,10 @@ bedrock_embedding_models: List = [
]
all_embedding_models = (
- open_ai_embedding_models + cohere_embedding_models + bedrock_embedding_models
+ open_ai_embedding_models
+ + cohere_embedding_models
+ + bedrock_embedding_models
+ + vertex_embedding_models
)
####### IMAGE GENERATION MODELS ###################
@@ -534,6 +564,7 @@ from .llms.bedrock import (
AmazonAnthropicConfig,
AmazonCohereConfig,
AmazonLlamaConfig,
+ AmazonStabilityConfig,
)
from .llms.openai import OpenAIConfig, OpenAITextCompletionConfig
from .llms.azure import AzureOpenAIConfig, AzureOpenAIError
diff --git a/litellm/_logging.py b/litellm/_logging.py
index e9a4a99cd16..438fa9743de 100644
--- a/litellm/_logging.py
+++ b/litellm/_logging.py
@@ -7,8 +7,11 @@ handler = logging.StreamHandler()
handler.setLevel(logging.DEBUG)
# Create a formatter and set it for the handler
+formatter = logging.Formatter(
+ "\033[92m%(asctime)s - %(name)s:%(levelname)s\033[0m: %(message)s",
+ datefmt="%H:%M:%S",
+)
-formatter = logging.Formatter("\033[92m%(name)s - %(levelname)s\033[0m: %(message)s")
handler.setFormatter(formatter)
diff --git a/litellm/_redis.py b/litellm/_redis.py
index bee73f13493..4484926d4b5 100644
--- a/litellm/_redis.py
+++ b/litellm/_redis.py
@@ -11,6 +11,7 @@
import os
import inspect
import redis, litellm
+import redis.asyncio as async_redis
from typing import List, Optional
@@ -67,7 +68,10 @@ def get_redis_url_from_environment():
)
-def get_redis_client(**env_overrides):
+def _get_redis_client_logic(**env_overrides):
+ """
+ Common functionality across sync + async redis client implementations
+ """
### check if "os.environ/