@@ -28,6 +28,8 @@ LiteLLM manages:
- Translate inputs to provider's `completion`, `embedding`, and `image_generation` endpoints
- [Consistent output](https://docs.litellm.ai/docs/completion/output), text responses will always be available at `['choices'][0]['message']['content']`
- Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing)
+- Track spend & set budgets per project [OpenAI Proxy Server](https://docs.litellm.ai/docs/simple_proxy)
+
[**Jump to OpenAI Proxy Docs**](https://github.com/BerriAI/litellm?tab=readme-ov-file#openai-proxy---docs) [**Jump to Supported LLM Providers**](https://github.com/BerriAI/litellm?tab=readme-ov-file#supported-provider-docs) @@ -155,6 +157,9 @@ print(response) ``` ## Proxy Key Management ([Docs](https://docs.litellm.ai/docs/proxy/virtual_keys)) +UI on `/ui` on your proxy server + + Track Spend, Set budgets and create virtual keys for the proxy `POST /key/generate` @@ -174,13 +179,6 @@ curl 'http://0.0.0.0:8000/key/generate' \ } ``` -### [Beta] Proxy UI ([Docs](https://docs.litellm.ai/docs/proxy/ui)) - -A UI to create keys, track spend per key - -Code: https://github.com/BerriAI/litellm/tree/main/ui - - ## Supported Providers ([Docs](https://docs.litellm.ai/docs/providers)) | Provider | [Completion](https://docs.litellm.ai/docs/#basic-usage) | [Streaming](https://docs.litellm.ai/docs/completion/stream#streaming-responses) | [Async Completion](https://docs.litellm.ai/docs/completion/stream#async-completion) | [Async Streaming](https://docs.litellm.ai/docs/completion/stream#async-streaming) | [Async Embedding](https://docs.litellm.ai/docs/embedding/supported_embedding) | [Async Image Generation](https://docs.litellm.ai/docs/image_generation) | | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | diff --git a/cookbook/litellm_router_load_test/test_loadtest_openai_client.py b/cookbook/litellm_router_load_test/test_loadtest_openai_client.py new file mode 100644 index 00000000000..d11249cd282 --- /dev/null +++ b/cookbook/litellm_router_load_test/test_loadtest_openai_client.py @@ -0,0 +1,76 @@ +import sys, os +import traceback +from dotenv import load_dotenv +import copy + +load_dotenv() +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path +import asyncio +from litellm import Router, Timeout +import time +from litellm.caching import Cache +import litellm +import openai + +### Test just calling AsyncAzureOpenAI + +openai_client = openai.AsyncAzureOpenAI( + azure_endpoint=os.getenv("AZURE_API_BASE"), + api_key=os.getenv("AZURE_API_KEY"), +) + + +async def call_acompletion(semaphore, input_data): + async with semaphore: + try: + # Use asyncio.wait_for to set a timeout for the task + response = await openai_client.chat.completions.create(**input_data) + # Handle the response as needed + print(response) + return response + except Timeout: + print(f"Task timed out: {input_data}") + return None # You may choose to return something else or raise an exception + + +async def main(): + # Initialize the Router + + # Create a semaphore with a capacity of 100 + semaphore = asyncio.Semaphore(100) + + # List to hold all task references + tasks = [] + start_time_all_tasks = time.time() + # Launch 1000 tasks + for _ in range(500): + task = asyncio.create_task( + call_acompletion( + semaphore, + { + "model": "chatgpt-v-2", + "messages": [{"role": "user", "content": "Hey, how's it going?"}], + }, + ) + ) + tasks.append(task) + + # Wait for all tasks to complete + responses = await asyncio.gather(*tasks) + # Process responses as needed + # Record the end time for all tasks + end_time_all_tasks = time.time() + # Calculate the total time for all tasks + total_time_all_tasks = end_time_all_tasks - start_time_all_tasks + print(f"Total time for all tasks: {total_time_all_tasks} seconds") + + # Calculate the average time per response + average_time_per_response = total_time_all_tasks / len(responses) + print(f"Average time per response: {average_time_per_response} seconds") + print(f"NUMBER OF COMPLETED TASKS: {len(responses)}") + + +# Run the main function +asyncio.run(main()) diff --git a/cookbook/litellm_router_load_test/test_loadtest_router.py b/cookbook/litellm_router_load_test/test_loadtest_router.py new file mode 100644 index 00000000000..a44bf4ccbbe --- /dev/null +++ b/cookbook/litellm_router_load_test/test_loadtest_router.py @@ -0,0 +1,88 @@ +import sys, os +import traceback +from dotenv import load_dotenv +import copy + +load_dotenv() +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path +import asyncio +from litellm import Router, Timeout +import time + +### Test calling router async + + +async def call_acompletion(semaphore, router: Router, input_data): + async with semaphore: + try: + # Use asyncio.wait_for to set a timeout for the task + response = await router.acompletion(**input_data) + # Handle the response as needed + print(response) + return response + except Timeout: + print(f"Task timed out: {input_data}") + return None # You may choose to return something else or raise an exception + + +async def main(): + # Initialize the Router + model_list = [ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": { + "model": "gpt-3.5-turbo", + "api_key": os.getenv("OPENAI_API_KEY"), + }, + }, + { + "model_name": "gpt-3.5-turbo", + "litellm_params": { + "model": "azure/chatgpt-v-2", + "api_key": os.getenv("AZURE_API_KEY"), + "api_base": os.getenv("AZURE_API_BASE"), + "api_version": os.getenv("AZURE_API_VERSION"), + }, + }, + ] + router = Router(model_list=model_list, num_retries=3, timeout=10) + + # Create a semaphore with a capacity of 100 + semaphore = asyncio.Semaphore(100) + + # List to hold all task references + tasks = [] + start_time_all_tasks = time.time() + # Launch 1000 tasks + for _ in range(500): + task = asyncio.create_task( + call_acompletion( + semaphore, + router, + { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hey, how's it going?"}], + }, + ) + ) + tasks.append(task) + + # Wait for all tasks to complete + responses = await asyncio.gather(*tasks) + # Process responses as needed + # Record the end time for all tasks + end_time_all_tasks = time.time() + # Calculate the total time for all tasks + total_time_all_tasks = end_time_all_tasks - start_time_all_tasks + print(f"Total time for all tasks: {total_time_all_tasks} seconds") + + # Calculate the average time per response + average_time_per_response = total_time_all_tasks / len(responses) + print(f"Average time per response: {average_time_per_response} seconds") + print(f"NUMBER OF COMPLETED TASKS: {len(responses)}") + + +# Run the main function +asyncio.run(main()) diff --git a/cookbook/litellm_router_load_test/test_loadtest_router_withs3_cache.py b/cookbook/litellm_router_load_test/test_loadtest_router_withs3_cache.py new file mode 100644 index 00000000000..5e6dd218c17 --- /dev/null +++ b/cookbook/litellm_router_load_test/test_loadtest_router_withs3_cache.py @@ -0,0 +1,94 @@ +import sys, os +import traceback +from dotenv import load_dotenv +import copy + +load_dotenv() +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path +import asyncio +from litellm import Router, Timeout +import time +from litellm.caching import Cache +import litellm + +litellm.cache = Cache( + type="s3", s3_bucket_name="cache-bucket-litellm", s3_region_name="us-west-2" +) + +### Test calling router with s3 Cache + + +async def call_acompletion(semaphore, router: Router, input_data): + async with semaphore: + try: + # Use asyncio.wait_for to set a timeout for the task + response = await router.acompletion(**input_data) + # Handle the response as needed + print(response) + return response + except Timeout: + print(f"Task timed out: {input_data}") + return None # You may choose to return something else or raise an exception + + +async def main(): + # Initialize the Router + model_list = [ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": { + "model": "gpt-3.5-turbo", + "api_key": os.getenv("OPENAI_API_KEY"), + }, + }, + { + "model_name": "gpt-3.5-turbo", + "litellm_params": { + "model": "azure/chatgpt-v-2", + "api_key": os.getenv("AZURE_API_KEY"), + "api_base": os.getenv("AZURE_API_BASE"), + "api_version": os.getenv("AZURE_API_VERSION"), + }, + }, + ] + router = Router(model_list=model_list, num_retries=3, timeout=10) + + # Create a semaphore with a capacity of 100 + semaphore = asyncio.Semaphore(100) + + # List to hold all task references + tasks = [] + start_time_all_tasks = time.time() + # Launch 1000 tasks + for _ in range(500): + task = asyncio.create_task( + call_acompletion( + semaphore, + router, + { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hey, how's it going?"}], + }, + ) + ) + tasks.append(task) + + # Wait for all tasks to complete + responses = await asyncio.gather(*tasks) + # Process responses as needed + # Record the end time for all tasks + end_time_all_tasks = time.time() + # Calculate the total time for all tasks + total_time_all_tasks = end_time_all_tasks - start_time_all_tasks + print(f"Total time for all tasks: {total_time_all_tasks} seconds") + + # Calculate the average time per response + average_time_per_response = total_time_all_tasks / len(responses) + print(f"Average time per response: {average_time_per_response} seconds") + print(f"NUMBER OF COMPLETED TASKS: {len(responses)}") + + +# Run the main function +asyncio.run(main()) diff --git a/docs/my-website/docs/enterprise.md b/docs/my-website/docs/enterprise.md new file mode 100644 index 00000000000..d530568d76c --- /dev/null +++ b/docs/my-website/docs/enterprise.md @@ -0,0 +1,15 @@ +# Enterprise + +LiteLLM offers dedicated enterprise support. + +This covers: +- **Feature Prioritization** +- **Custom Integrations** +- **Professional Support - Dedicated discord + slack** +- **Custom SLAs** + +:::info + +[Talk to founders](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) + +::: \ No newline at end of file diff --git a/docs/my-website/docs/index.md b/docs/my-website/docs/index.md index f6bd7bc42b6..d7ed1401950 100644 --- a/docs/my-website/docs/index.md +++ b/docs/my-website/docs/index.md @@ -5,10 +5,14 @@ import TabItem from '@theme/TabItem'; https://github.com/BerriAI/litellm -import QuickStart from '../src/components/QuickStart.js' ## **Call 100+ LLMs using the same Input/Output Format** +- Translate inputs to provider's `completion`, `embedding`, and `image_generation` endpoints +- [Consistent output](https://docs.litellm.ai/docs/completion/output), text responses will always be available at `['choices'][0]['message']['content']` +- Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing) +- Track spend & set budgets per project [OpenAI Proxy Server](https://docs.litellm.ai/docs/simple_proxy) + ## Basic usage
@@ -157,9 +161,6 @@ response = completion(
messages=[{ "content": "Hello, how are you?","role": "user"}],
stream=True,
)
-
-for chunk in response:
- print(chunk)
```
@@ -177,9 +178,6 @@ response = completion(
messages=[{ "content": "Hello, how are you?","role": "user"}],
stream=True,
)
-
-for chunk in response:
- print(chunk)
```
@@ -199,9 +197,6 @@ response = completion(
messages=[{ "content": "Hello, how are you?","role": "user"}],
stream=True,
)
-
-for chunk in response:
- print(chunk)
```
@@ -222,9 +217,7 @@ response = completion(
stream=True,
)
-
-for chunk in response:
- print(chunk)
+print(response)
```
@@ -246,9 +239,6 @@ response = completion(
messages = [{ "content": "Hello, how are you?","role": "user"}],
stream=True,
)
-
-for chunk in response:
- print(chunk)
```
@@ -265,9 +255,6 @@ response = completion(
api_base="http://localhost:11434",
stream=True,
)
-
-for chunk in response:
- print(chunk)
```
@@ -306,30 +311,7 @@ litellm.success_callback = ["langfuse", "llmonitor"] # log input/output to langf
response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}])
```
-## Calculate Costs, Usage, Latency
-
-Pass the completion response to `litellm.completion_cost(completion_response=response)` and get the cost
-
-```python
-from litellm import completion, completion_cost
-import os
-os.environ["OPENAI_API_KEY"] = "your-api-key"
-
-response = completion(
- model="gpt-3.5-turbo",
- messages=[{ "content": "Hello, how are you?","role": "user"}]
-)
-
-cost = completion_cost(completion_response=response)
-print("Cost for completion call with gpt-3.5-turbo: ", f"${float(cost):.10f}")
-```
-
-**Output**
-```shell
-Cost for completion call with gpt-3.5-turbo: $0.0000775000
-```
-
-### Track Costs, Usage, Latency for streaming
+## Track Costs, Usage, Latency for streaming
Use a callback function for this - more info on custom callbacks: https://docs.litellm.ai/docs/observability/custom_callback
```python
@@ -342,18 +324,8 @@ def track_cost_callback(
start_time, end_time # start/end time
):
try:
- # check if it has collected an entire stream response
- if "complete_streaming_response" in kwargs:
- # for tracking streaming cost we pass the "messages" and the output_text to litellm.completion_cost
- completion_response=kwargs["complete_streaming_response"]
- input_text = kwargs["messages"]
- output_text = completion_response["choices"][0]["message"]["content"]
- response_cost = litellm.completion_cost(
- model = kwargs["model"],
- messages = input_text,
- completion=output_text
- )
- print("streaming response_cost", response_cost)
+ response_cost = kwargs.get("response_cost", 0)
+ print("streaming response_cost", response_cost)
except:
pass
# set callback
@@ -372,13 +344,12 @@ response = completion(
)
```
-
-Need a dedicated key? Email us @ krrish@berri.ai
-
## OpenAI Proxy
Track spend across multiple projects/people
+
+
The proxy provides:
1. [Hooks for auth](https://docs.litellm.ai/docs/proxy/virtual_keys#custom-auth)
2. [Hooks for logging](https://docs.litellm.ai/docs/proxy/logging#step-1---create-your-custom-litellm-callback-class)
@@ -418,4 +389,4 @@ print(response)
## More details
* [exception mapping](./exception_mapping.md)
* [retries + model fallbacks for completion()](./completion/reliable_completions.md)
-* [tutorial for model fallbacks with completion()](./tutorials/fallbacks.md)
\ No newline at end of file
+* [proxy virtual keys & spend management](./tutorials/fallbacks.md)
\ No newline at end of file
diff --git a/litellm/caching.py b/litellm/caching.py
index f996a587354..f0ae7778af9 100644
--- a/litellm/caching.py
+++ b/litellm/caching.py
@@ -427,10 +427,16 @@ class RedisSemanticCache(BaseCache):
else []
)
if llm_router is not None and self.embedding_model in router_model_names:
+ user_api_key = kwargs.get("metadata", {}).get("user_api_key", "")
embedding_response = await llm_router.aembedding(
model=self.embedding_model,
input=prompt,
cache={"no-store": True, "no-cache": True},
+ metadata={
+ "user_api_key": user_api_key,
+ "semantic-cache-embedding": True,
+ "trace_id": kwargs.get("metadata", {}).get("trace_id", None),
+ },
)
else:
# convert to embedding
@@ -476,10 +482,16 @@ class RedisSemanticCache(BaseCache):
else []
)
if llm_router is not None and self.embedding_model in router_model_names:
+ user_api_key = kwargs.get("metadata", {}).get("user_api_key", "")
embedding_response = await llm_router.aembedding(
model=self.embedding_model,
input=prompt,
cache={"no-store": True, "no-cache": True},
+ metadata={
+ "user_api_key": user_api_key,
+ "semantic-cache-embedding": True,
+ "trace_id": kwargs.get("metadata", {}).get("trace_id", None),
+ },
)
else:
# convert to embedding
diff --git a/litellm/integrations/langfuse.py b/litellm/integrations/langfuse.py
index bb49cb048af..19714b76396 100644
--- a/litellm/integrations/langfuse.py
+++ b/litellm/integrations/langfuse.py
@@ -266,37 +266,33 @@ class LangFuseLogger:
trace = self.Langfuse.trace(**trace_params)
- if level == "ERROR":
- trace.generation(
- level="ERROR", # can be any of DEBUG, DEFAULT, WARNING or ERROR
- status_message=output, # can be any string (e.g. stringified stack trace or error body)
- )
- print(f"SUCCESSFULLY LOGGED ERROR")
- else:
- # get generation_id
- generation_id = None
- if (
- response_obj is not None
- and response_obj.get("id", None) is not None
- ):
- generation_id = litellm.utils.get_logging_id(
- start_time, response_obj
- )
- trace.generation(
- name=generation_name,
- id=metadata.get("generation_id", generation_id),
- startTime=start_time,
- endTime=end_time,
- model=kwargs["model"],
- modelParameters=optional_params,
- input=input,
- output=output,
- usage={
- "prompt_tokens": response_obj["usage"]["prompt_tokens"],
- "completion_tokens": response_obj["usage"]["completion_tokens"],
- "total_cost": cost if supports_costs else None,
- },
- metadata=metadata,
- )
+ generation_id = None
+ usage = None
+ if response_obj is not None and response_obj.get("id", None) is not None:
+ generation_id = litellm.utils.get_logging_id(start_time, response_obj)
+ usage = {
+ "prompt_tokens": response_obj["usage"]["prompt_tokens"],
+ "completion_tokens": response_obj["usage"]["completion_tokens"],
+ "total_cost": cost if supports_costs else None,
+ }
+
+ generation_params = {
+ "name": generation_name,
+ "id": metadata.get("generation_id", generation_id),
+ "startTime": start_time,
+ "endTime": end_time,
+ "model": kwargs["model"],
+ "modelParameters": optional_params,
+ "input": input,
+ "output": output,
+ "usage": usage,
+ "metadata": metadata,
+ "level": level,
+ }
+
+ if output is not None and isinstance(output, str) and level == "ERROR":
+ generation_params["statusMessage"] = output
+
+ trace.generation(**generation_params)
except Exception as e:
print(f"Langfuse Layer Error - {traceback.format_exc()}")
diff --git a/litellm/llms/bedrock.py b/litellm/llms/bedrock.py
index 0fe9b72e385..4118e3183a0 100644
--- a/litellm/llms/bedrock.py
+++ b/litellm/llms/bedrock.py
@@ -356,6 +356,7 @@ def init_bedrock_client(
aws_region_name: Optional[str] = None,
aws_bedrock_runtime_endpoint: Optional[str] = None,
aws_session_name: Optional[str] = None,
+ aws_profile_name: Optional[str] = None,
aws_role_name: Optional[str] = None,
timeout: Optional[int] = None,
):
@@ -371,6 +372,7 @@ def init_bedrock_client(
aws_region_name,
aws_bedrock_runtime_endpoint,
aws_session_name,
+ aws_profile_name,
aws_role_name,
]
@@ -385,6 +387,7 @@ def init_bedrock_client(
aws_region_name,
aws_bedrock_runtime_endpoint,
aws_session_name,
+ aws_profile_name,
aws_role_name,
) = params_to_check
@@ -450,6 +453,15 @@ def init_bedrock_client(
endpoint_url=endpoint_url,
config=config,
)
+ elif aws_profile_name is not None:
+ # uses auth values from AWS profile usually stored in ~/.aws/credentials
+
+ client = boto3.Session(profile_name=aws_profile_name).client(
+ service_name="bedrock-runtime",
+ region_name=region_name,
+ endpoint_url=endpoint_url,
+ config=config,
+ )
else:
# aws_access_key_id is None, assume user is trying to auth using env variables
# boto3 automatically reads env variables
@@ -523,6 +535,7 @@ def completion(
aws_region_name = optional_params.pop("aws_region_name", None)
aws_role_name = optional_params.pop("aws_role_name", None)
aws_session_name = optional_params.pop("aws_session_name", None)
+ aws_profile_name = optional_params.pop("aws_profile_name", None)
aws_bedrock_runtime_endpoint = optional_params.pop(
"aws_bedrock_runtime_endpoint", None
)
@@ -539,6 +552,7 @@ def completion(
aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
aws_role_name=aws_role_name,
aws_session_name=aws_session_name,
+ aws_profile_name=aws_profile_name,
)
model = model
@@ -725,6 +739,8 @@ def completion(
try:
if len(outputText) > 0:
model_response["choices"][0]["message"]["content"] = outputText
+ else:
+ raise Exception()
except:
raise BedrockError(
message=json.dumps(outputText),
diff --git a/litellm/llms/openai.py b/litellm/llms/openai.py
index 3f151d1a92a..1ca7e1710f7 100644
--- a/litellm/llms/openai.py
+++ b/litellm/llms/openai.py
@@ -222,6 +222,7 @@ class OpenAIChatCompletion(BaseLLM):
custom_prompt_dict: dict = {},
client=None,
organization: Optional[str] = None,
+ custom_llm_provider: Optional[str] = None,
):
super().completion()
exception_mapping_worked = False
@@ -236,6 +237,14 @@ class OpenAIChatCompletion(BaseLLM):
status_code=422, message=f"Timeout needs to be a float"
)
+ if custom_llm_provider == "mistral":
+ # check if message content passed in as list, and not string
+ messages = prompt_factory(
+ model=model,
+ messages=messages,
+ custom_llm_provider=custom_llm_provider,
+ )
+
for _ in range(
2
): # if call fails due to alternating messages, retry with reformatted message
@@ -325,12 +334,13 @@ class OpenAIChatCompletion(BaseLLM):
model_response_object=model_response,
)
except Exception as e:
- if "Conversation roles must alternate user/assistant" in str(
- e
- ) or "user and assistant roles should be alternating" in str(e):
+ if (
+ "Conversation roles must alternate user/assistant" in str(e)
+ or "user and assistant roles should be alternating" in str(e)
+ ) and messages is not None:
# reformat messages to ensure user/assistant are alternating, if there's either 2 consecutive 'user' messages or 2 consecutive 'assistant' message, add a blank 'user' or 'assistant' message to ensure compatibility
new_messages = []
- for i in range(len(messages) - 1):
+ for i in range(len(messages) - 1): # type: ignore
new_messages.append(messages[i])
if messages[i]["role"] == messages[i + 1]["role"]:
if messages[i]["role"] == "user":
@@ -341,7 +351,9 @@ class OpenAIChatCompletion(BaseLLM):
new_messages.append({"role": "user", "content": ""})
new_messages.append(messages[-1])
messages = new_messages
- elif "Last message must have role `user`" in str(e):
+ elif (
+ "Last message must have role `user`" in str(e)
+ ) and messages is not None:
new_messages = messages
new_messages.append({"role": "user", "content": ""})
messages = new_messages
diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py
index 1aebcf35d06..6321860cc5b 100644
--- a/litellm/llms/prompt_templates/factory.py
+++ b/litellm/llms/prompt_templates/factory.py
@@ -116,6 +116,28 @@ def mistral_instruct_pt(messages):
return prompt
+def mistral_api_pt(messages):
+ """
+ - handles scenario where content is list and not string
+ - content list is just text, and no images
+ - if image passed in, then just return as is (user-intended)
+
+ Motivation: mistral api doesn't support content as a list
+ """
+ new_messages = []
+ for m in messages:
+ texts = ""
+ if isinstance(m["content"], list):
+ for c in m["content"]:
+ if c["type"] == "image_url":
+ return messages
+ elif c["type"] == "text" and isinstance(c["text"], str):
+ texts += c["text"]
+ new_m = {"role": m["role"], "content": texts}
+ new_messages.append(new_m)
+ return new_messages
+
+
# Falcon prompt template - from https://github.com/lm-sys/FastChat/blob/main/fastchat/conversation.py#L110
def falcon_instruct_pt(messages):
prompt = ""
@@ -612,6 +634,8 @@ def prompt_factory(
return _gemini_vision_convert_messages(messages=messages)
else:
return gemini_text_image_pt(messages=messages)
+ elif custom_llm_provider == "mistral":
+ return mistral_api_pt(messages=messages)
try:
if "meta-llama/llama-2" in model and "chat" in model:
return llama_2_chat_pt(messages=messages)
diff --git a/litellm/main.py b/litellm/main.py
index 384dadc32d5..d11e76048f6 100644
--- a/litellm/main.py
+++ b/litellm/main.py
@@ -10,7 +10,6 @@
import os, openai, sys, json, inspect, uuid, datetime, threading
from typing import Any, Literal, Union
from functools import partial
-
import dotenv, traceback, random, asyncio, time, contextvars
from copy import deepcopy
import httpx
@@ -31,6 +30,7 @@ from litellm.utils import (
get_llm_provider,
get_api_key,
mock_completion_streaming_obj,
+ async_mock_completion_streaming_obj,
convert_to_model_response_object,
token_counter,
Usage,
@@ -307,6 +307,7 @@ def mock_completion(
messages: List,
stream: Optional[bool] = False,
mock_response: str = "This is a mock request",
+ logging=None,
**kwargs,
):
"""
@@ -335,6 +336,15 @@ def mock_completion(
model_response = ModelResponse(stream=stream)
if stream is True:
# don't try to access stream object,
+ if kwargs.get("acompletion", False) == True:
+ return CustomStreamWrapper(
+ completion_stream=async_mock_completion_streaming_obj(
+ model_response, mock_response=mock_response, model=model
+ ),
+ model=model,
+ custom_llm_provider="openai",
+ logging_obj=logging,
+ )
response = mock_completion_streaming_obj(
model_response, mock_response=mock_response, model=model
)
@@ -717,7 +727,12 @@ def completion(
)
if mock_response:
return mock_completion(
- model, messages, stream=stream, mock_response=mock_response
+ model,
+ messages,
+ stream=stream,
+ mock_response=mock_response,
+ logging=logging,
+ acompletion=acompletion,
)
if custom_llm_provider == "azure":
# azure configs
diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index 4c28bdbe8b3..e44f46ff84a 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -8,6 +8,15 @@
"litellm_provider": "openai",
"mode": "chat"
},
+ "gpt-4-turbo-preview": {
+ "max_tokens": 8192,
+ "max_input_tokens": 8192,
+ "max_output_tokens": 4096,
+ "input_cost_per_token": 0.00001,
+ "output_cost_per_token": 0.00003,
+ "litellm_provider": "openai",
+ "mode": "chat"
+ },
"gpt-4-0314": {
"max_tokens": 8192,
"max_input_tokens": 8192,
@@ -243,6 +252,15 @@
"output_cost_per_pixel": 0.0,
"litellm_provider": "openai"
},
+ "azure/gpt-4-0125-preview": {
+ "max_tokens": 128000,
+ "max_input_tokens": 128000,
+ "max_output_tokens": 4096,
+ "input_cost_per_token": 0.00001,
+ "output_cost_per_token": 0.00003,
+ "litellm_provider": "azure",
+ "mode": "chat"
+ },
"azure/gpt-4-1106-preview": {
"max_tokens": 128000,
"max_input_tokens": 128000,
diff --git a/litellm/proxy/_experimental/out/404.html b/litellm/proxy/_experimental/out/404.html
index c57eb5193a2..74bcfb57bf0 100644
--- a/litellm/proxy/_experimental/out/404.html
+++ b/litellm/proxy/_experimental/out/404.html
@@ -1 +1 @@
-404
This page could not be found.
404
This page could not be found.
Loading...
\ No newline at end of file
+Loading...
\ No newline at end of file
diff --git a/litellm/proxy/_experimental/out/index.txt b/litellm/proxy/_experimental/out/index.txt
index f48954f2c46..97964f308ee 100644
--- a/litellm/proxy/_experimental/out/index.txt
+++ b/litellm/proxy/_experimental/out/index.txt
@@ -1,7 +1,7 @@
2:"$Sreact.suspense"
-3:I[27680,["787","static/chunks/787-5bb33960644f5c7c.js","931","static/chunks/app/page-7f03ccc8529ada97.js"],""]
+3:I[42989,["787","static/chunks/787-5bb33960644f5c7c.js","931","static/chunks/app/page-b1e8aae39e9306e9.js"],""]
4:I[5613,[],""]
5:I[31778,[],""]
-0:["p5gDwQBbgW8D3Uz3lgoZg",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},["$L1",["$","$2",null,{"fallback":["$","div",null,{"children":"Loading..."}],"children":["$","div",null,{"className":"flex min-h-screen flex-col ","children":["$","$L3",null,{}]}]}],null]]},[null,["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_c23dc8","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"loading":"$undefined","loadingStyles":"$undefined","loadingScripts":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[],"styles":null}]}]}],null]],[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/a6a9860a7fe022a9.css","precedence":"next","crossOrigin":""}]],"$L6"]]]]
+0:["Ul7bzKEbTPs-NwnRJpbmM",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},["$L1",["$","$2",null,{"fallback":["$","div",null,{"children":"Loading..."}],"children":["$","div",null,{"className":"flex min-h-screen flex-col ","children":["$","$L3",null,{}]}]}],null]]},[null,["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_c23dc8","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"loading":"$undefined","loadingStyles":"$undefined","loadingScripts":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[],"styles":null}]}]}],null]],[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/173c5a76ba57a3b8.css","precedence":"next","crossOrigin":""}]],"$L6"]]]]
6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"🚅 LiteLLM"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/ui/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","meta","5",{"name":"next-size-adjust"}]]
1:null
diff --git a/litellm/proxy/hooks/parallel_request_limiter.py b/litellm/proxy/hooks/parallel_request_limiter.py
index 48cf5b7799b..67f8d1ad2f4 100644
--- a/litellm/proxy/hooks/parallel_request_limiter.py
+++ b/litellm/proxy/hooks/parallel_request_limiter.py
@@ -125,7 +125,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
# ------------
new_val = {
- "current_requests": current["current_requests"] - 1,
+ "current_requests": max(current["current_requests"] - 1, 0),
"current_tpm": current["current_tpm"] + total_tokens,
"current_rpm": current["current_rpm"] + 1,
}
@@ -183,7 +183,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
}
new_val = {
- "current_requests": current["current_requests"] - 1,
+ "current_requests": max(current["current_requests"] - 1, 0),
"current_tpm": current["current_tpm"],
"current_rpm": current["current_rpm"],
}
diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py
index 4982900948c..9f0ed8de174 100644
--- a/litellm/proxy/proxy_cli.py
+++ b/litellm/proxy/proxy_cli.py
@@ -424,14 +424,25 @@ def run_server(
if database_url is not None and isinstance(database_url, str):
os.environ["DATABASE_URL"] = database_url
- if os.getenv("DATABASE_URL", None) is not None:
+ if (
+ os.getenv("DATABASE_URL", None) is not None
+ or os.getenv("DIRECT_URL", None) is not None
+ ):
try:
- ### add connection pool + pool timeout args
- params = {"connection_limit": 500, "pool_timeout": 60}
- database_url = os.getenv("DATABASE_URL")
- modified_url = append_query_params(database_url, params)
- os.environ["DATABASE_URL"] = modified_url
- ###
+ if os.getenv("DATABASE_URL", None) is not None:
+ ### add connection pool + pool timeout args
+ params = {"connection_limit": 100, "pool_timeout": 60}
+ database_url = os.getenv("DATABASE_URL")
+ modified_url = append_query_params(database_url, params)
+ os.environ["DATABASE_URL"] = modified_url
+ ###
+ if os.getenv("DIRECT_URL", None) is not None:
+ ### add connection pool + pool timeout args
+ params = {"connection_limit": 100, "pool_timeout": 60}
+ database_url = os.getenv("DIRECT_URL")
+ modified_url = append_query_params(database_url, params)
+ os.environ["DIRECT_URL"] = modified_url
+ ###
subprocess.run(["prisma"], capture_output=True)
is_prisma_runnable = True
except FileNotFoundError:
diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml
index 7d774d9105c..82f0fa877f6 100644
--- a/litellm/proxy/proxy_config.yaml
+++ b/litellm/proxy/proxy_config.yaml
@@ -7,10 +7,8 @@ model_list:
api_version: "2023-07-01-preview"
model_info:
mode: chat
- input_cost_per_token: 0.0.00006
- output_cost_per_token: 0.00003
max_tokens: 4096
- base_model: gpt-3.5-turbo
+ base_model: azure/gpt-4-1106-preview
- model_name: gpt-4
litellm_params:
model: azure/chatgpt-v-2
@@ -74,11 +72,6 @@ litellm_settings:
max_budget: 1.5000
models: ["azure-gpt-3.5"]
duration: None
- cache: True # set cache responses to True
- cache_params:
- type: "redis-semantic"
- similarity_threshold: 0.8
- redis_semantic_cache_embedding_model: azure-embedding-model
upperbound_key_generate_params:
max_budget: 100
duration: "30d"
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index b57f7808956..a2d03486da0 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -809,16 +809,15 @@ async def _PROXY_track_cost_callback(
litellm_params = kwargs.get("litellm_params", {}) or {}
proxy_server_request = litellm_params.get("proxy_server_request") or {}
user_id = proxy_server_request.get("body", {}).get("user", None)
+ user_id = user_id or kwargs["litellm_params"]["metadata"].get(
+ "user_api_key_user_id", None
+ )
if kwargs.get("response_cost", None) is not None:
response_cost = kwargs["response_cost"]
user_api_key = kwargs["litellm_params"]["metadata"].get(
"user_api_key", None
)
- user_id = user_id or kwargs["litellm_params"]["metadata"].get(
- "user_api_key_user_id", None
- )
-
if kwargs.get("cache_hit", False) == True:
response_cost = 0.0
verbose_proxy_logger.info(
@@ -852,9 +851,21 @@ async def _PROXY_track_cost_callback(
f"Model not in litellm model cost map. Add custom pricing - https://docs.litellm.ai/docs/proxy/custom_pricing"
)
except Exception as e:
- verbose_proxy_logger.debug(
- f"error in tracking cost callback - {traceback.format_exc}"
+ error_msg = f"error in tracking cost callback - {traceback.format_exc()}"
+ model = kwargs.get("model", "")
+ metadata = kwargs.get("litellm_params", {}).get("metadata", {})
+ error_msg += f"\n Args to _PROXY_track_cost_callback\n model: {model}\n metadata: {metadata}\n"
+ user_id = user_id or "not-found"
+ asyncio.create_task(
+ proxy_logging_obj.budget_alerts(
+ user_max_budget=0,
+ user_current_spend=0,
+ type="failed_tracking",
+ user_info=user_id,
+ error_message=error_msg,
+ )
)
+ verbose_proxy_logger.debug(f"error in tracking cost callback - {error_msg}")
async def update_database(
@@ -2004,12 +2015,15 @@ async def startup_event():
)
### START BUDGET SCHEDULER ###
- scheduler = AsyncIOScheduler()
- interval = random.randint(
- 597, 605
- ) # random interval, so multiple workers avoid resetting budget at the same time
- scheduler.add_job(reset_budget, "interval", seconds=interval, args=[prisma_client])
- scheduler.start()
+ if prisma_client is not None:
+ scheduler = AsyncIOScheduler()
+ interval = random.randint(
+ 597, 605
+ ) # random interval, so multiple workers avoid resetting budget at the same time
+ scheduler.add_job(
+ reset_budget, "interval", seconds=interval, args=[prisma_client]
+ )
+ scheduler.start()
#### API ENDPOINTS ####
@@ -3001,7 +3015,16 @@ async def info_key_fn(
tags=["budget & spend Tracking"],
dependencies=[Depends(user_api_key_auth)],
)
-async def spend_key_fn():
+async def spend_key_fn(
+ start_date: Optional[str] = fastapi.Query(
+ default=None,
+ description="Time from which to start viewing key spend",
+ ),
+ end_date: Optional[str] = fastapi.Query(
+ default=None,
+ description="Time till which to view key spend",
+ ),
+):
"""
View all keys created, ordered by spend
@@ -3018,9 +3041,41 @@ async def spend_key_fn():
f"Database not connected. Connect a database to your proxy - https://docs.litellm.ai/docs/simple_proxy#managing-auth---virtual-keys"
)
- key_info = await prisma_client.get_data(table_name="key", query_type="find_all")
+ if (
+ start_date is not None
+ and isinstance(start_date, str)
+ and end_date is not None
+ and isinstance(end_date, str)
+ ):
+ # Convert the date strings to datetime objects
+ start_date_obj = datetime.strptime(start_date, "%Y-%m-%d")
+ end_date_obj = datetime.strptime(end_date, "%Y-%m-%d")
- return key_info
+ # SQL query
+ response = await prisma_client.db.litellm_spendlogs.group_by(
+ by=["api_key", "startTime"],
+ where={
+ "startTime": {
+ "gte": start_date_obj, # Greater than or equal to Start Date
+ "lte": end_date_obj, # Less than or equal to End Date
+ }
+ },
+ sum={
+ "spend": True,
+ },
+ )
+
+ # TODO: Execute SQL query and return the results
+
+ return {
+ "message": "This is your SQL query",
+ "response": response,
+ }
+ else:
+ key_info = await prisma_client.get_data(
+ table_name="key", query_type="find_all"
+ )
+ return key_info
except Exception as e:
raise HTTPException(
@@ -3441,10 +3496,25 @@ async def login(request: Request):
litellm_dashboard_ui = "/ui/"
+ user_role = "app_owner"
+ if (
+ os.getenv("PROXY_ADMIN_ID", None) is not None
+ and os.environ["PROXY_ADMIN_ID"] == user_id
+ ):
+ # checks if user is admin
+ user_role = "app_admin"
+
import jwt
jwt_token = jwt.encode(
- {"user_id": user_id, "key": key}, "secret", algorithm="HS256"
+ {
+ "user_id": user_id,
+ "key": key,
+ "user_email": user_id,
+ "user_role": user_role,
+ },
+ "secret",
+ algorithm="HS256",
)
litellm_dashboard_ui += "?userID=" + user_id + "&token=" + jwt_token
diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma
index 2164358ccc4..3a5c3ea230f 100644
--- a/litellm/proxy/schema.prisma
+++ b/litellm/proxy/schema.prisma
@@ -1,6 +1,7 @@
datasource client {
provider = "postgresql"
url = env("DATABASE_URL")
+ directUrl = env("DIRECT_URL")
}
generator client {
diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index 8804028cc11..919799ca586 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -198,10 +198,17 @@ class ProxyLogging:
async def budget_alerts(
self,
- type: Literal["token_budget", "user_budget", "user_and_proxy_budget"],
+ type: Literal[
+ "token_budget",
+ "user_budget",
+ "user_and_proxy_budget",
+ "failed_budgets",
+ "failed_tracking",
+ ],
user_max_budget: float,
user_current_spend: float,
user_info=None,
+ error_message="",
):
if self.alerting is None:
# do nothing if alerting is not switched on
@@ -221,6 +228,15 @@ class ProxyLogging:
max_budget = token_info["max_budget"]
user_id = token_info["user_id"]
user_info = f"""\nToken: {token}\nSpend: ${spend}\nMax Budget: ${max_budget}\nUser ID: {user_id}"""
+ elif type == "failed_tracking":
+ user_id = str(user_info)
+ user_info = f"\nUser ID: {user_id}\n Error {error_message}"
+ message = "Failed Tracking Cost for" + user_info
+ await self.alerting_handler(
+ message=message,
+ level="High",
+ )
+ return
else:
user_info = str(user_info)
# percent of max_budget left to spend
diff --git a/litellm/router.py b/litellm/router.py
index a970b5d449c..e9bf6dfed5c 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -304,7 +304,10 @@ class Router:
specific_deployment=kwargs.pop("specific_deployment", None),
)
kwargs.setdefault("metadata", {}).update(
- {"deployment": deployment["litellm_params"]["model"]}
+ {
+ "deployment": deployment["litellm_params"]["model"],
+ "model_info": deployment.get("model_info", {}),
+ }
)
data = deployment["litellm_params"].copy()
kwargs["model_info"] = deployment.get("model_info", {})
@@ -376,7 +379,10 @@ class Router:
specific_deployment=kwargs.pop("specific_deployment", None),
)
kwargs.setdefault("metadata", {}).update(
- {"deployment": deployment["litellm_params"]["model"]}
+ {
+ "deployment": deployment["litellm_params"]["model"],
+ "model_info": deployment.get("model_info", {}),
+ }
)
kwargs["model_info"] = deployment.get("model_info", {})
data = deployment["litellm_params"].copy()
@@ -451,7 +457,10 @@ class Router:
specific_deployment=kwargs.pop("specific_deployment", None),
)
kwargs.setdefault("metadata", {}).update(
- {"deployment": deployment["litellm_params"]["model"]}
+ {
+ "deployment": deployment["litellm_params"]["model"],
+ "model_info": deployment.get("model_info", {}),
+ }
)
kwargs["model_info"] = deployment.get("model_info", {})
data = deployment["litellm_params"].copy()
@@ -526,7 +535,10 @@ class Router:
specific_deployment=kwargs.pop("specific_deployment", None),
)
kwargs.setdefault("metadata", {}).update(
- {"deployment": deployment["litellm_params"]["model"]}
+ {
+ "deployment": deployment["litellm_params"]["model"],
+ "model_info": deployment.get("model_info", {}),
+ }
)
kwargs["model_info"] = deployment.get("model_info", {})
data = deployment["litellm_params"].copy()
@@ -654,7 +666,10 @@ class Router:
specific_deployment=kwargs.pop("specific_deployment", None),
)
kwargs.setdefault("metadata", {}).update(
- {"deployment": deployment["litellm_params"]["model"]}
+ {
+ "deployment": deployment["litellm_params"]["model"],
+ "model_info": deployment.get("model_info", {}),
+ }
)
kwargs["model_info"] = deployment.get("model_info", {})
data = deployment["litellm_params"].copy()
@@ -780,7 +795,10 @@ class Router:
specific_deployment=kwargs.pop("specific_deployment", None),
)
kwargs.setdefault("metadata", {}).update(
- {"deployment": deployment["litellm_params"]["model"]}
+ {
+ "deployment": deployment["litellm_params"]["model"],
+ "model_info": deployment.get("model_info", {}),
+ }
)
kwargs["model_info"] = deployment.get("model_info", {})
data = deployment["litellm_params"].copy()
@@ -1407,7 +1425,6 @@ class Router:
max_retries = litellm.get_secret(max_retries_env_name)
litellm_params["max_retries"] = max_retries
-
# proxy support
import os
import httpx
diff --git a/litellm/tests/test_amazing_s3_logs.py b/litellm/tests/test_amazing_s3_logs.py
index ecc6ae26f22..7b8eb4a47d7 100644
--- a/litellm/tests/test_amazing_s3_logs.py
+++ b/litellm/tests/test_amazing_s3_logs.py
@@ -66,6 +66,7 @@ def test_s3_logging():
response = asyncio.run(_test())
expected_keys.append(response.id)
print(f"response: {response}")
+ time.sleep(5) # wait 5s for logs to land
import boto3
@@ -121,6 +122,71 @@ def test_s3_logging():
# test_s3_logging()
+def test_s3_logging_async():
+ # this tests time added to make s3 logging calls, vs just acompletion calls
+ try:
+ litellm.set_verbose = True
+ # Make 5 calls with an empty success_callback
+ litellm.success_callback = []
+ start_time_empty_callback = asyncio.run(make_async_calls())
+ print("done with no callback test")
+
+ print("starting s3 logging load test")
+ # Make 5 calls with success_callback set to "langfuse"
+ litellm.success_callback = ["s3"]
+ litellm.s3_callback_params = {
+ "s3_bucket_name": "litellm-logs",
+ "s3_aws_secret_access_key": "os.environ/AWS_SECRET_ACCESS_KEY",
+ "s3_aws_access_key_id": "os.environ/AWS_ACCESS_KEY_ID",
+ }
+ start_time_s3 = asyncio.run(make_async_calls())
+ print("done with s3 test")
+
+ # Compare the time for both scenarios
+ print(f"Time taken with success_callback='s3': {start_time_s3}")
+ print(f"Time taken with empty success_callback: {start_time_empty_callback}")
+
+ # assert the diff is not more than 1 second
+ assert abs(start_time_s3 - start_time_empty_callback) < 1
+
+ except litellm.Timeout as e:
+ pass
+ except Exception as e:
+ pytest.fail(f"An exception occurred - {e}")
+
+
+async def make_async_calls():
+ tasks = []
+ for _ in range(5):
+ task = asyncio.create_task(
+ litellm.acompletion(
+ model="azure/chatgpt-v-2",
+ messages=[{"role": "user", "content": "This is a test"}],
+ max_tokens=5,
+ temperature=0.7,
+ timeout=5,
+ user="langfuse_latency_test_user",
+ mock_response="It's simple to use and easy to get started",
+ )
+ )
+ tasks.append(task)
+
+ # Measure the start time before running the tasks
+ start_time = asyncio.get_event_loop().time()
+
+ # Wait for all tasks to complete
+ responses = await asyncio.gather(*tasks)
+
+ # Print the responses when tasks return
+ for idx, response in enumerate(responses):
+ print(f"Response from Task {idx + 1}: {response}")
+
+ # Calculate the total time taken
+ total_time = asyncio.get_event_loop().time() - start_time
+
+ return total_time
+
+
def test_s3_logging_r2():
# all s3 requests need to be in one test function
# since we are modifying stdout, and pytests runs tests in parallel
diff --git a/litellm/tests/test_caching.py b/litellm/tests/test_caching.py
index 8433941e90f..5887aba12c0 100644
--- a/litellm/tests/test_caching.py
+++ b/litellm/tests/test_caching.py
@@ -696,6 +696,58 @@ def test_s3_cache_acompletion_stream_azure():
# test_s3_cache_acompletion_stream_azure()
+@pytest.mark.asyncio
+async def test_s3_cache_acompletion_azure():
+ import asyncio
+ import logging
+ import tracemalloc
+
+ tracemalloc.start()
+ logging.basicConfig(level=logging.DEBUG)
+
+ try:
+ litellm.set_verbose = True
+ random_word = generate_random_word()
+ messages = [
+ {
+ "role": "user",
+ "content": f"write a one sentence poem about: {random_word}",
+ }
+ ]
+ litellm.cache = Cache(
+ type="s3", s3_bucket_name="cache-bucket-litellm", s3_region_name="us-west-2"
+ )
+ print("s3 Cache: test for caching, streaming + completion")
+
+ response1 = await litellm.acompletion(
+ model="azure/chatgpt-v-2",
+ messages=messages,
+ max_tokens=40,
+ temperature=1,
+ )
+ print(response1)
+
+ time.sleep(2)
+
+ response2 = await litellm.acompletion(
+ model="azure/chatgpt-v-2",
+ messages=messages,
+ max_tokens=40,
+ temperature=1,
+ )
+
+ print(response2)
+
+ assert response1.id == response2.id
+
+ litellm.cache = None
+ litellm.success_callback = []
+ litellm._async_success_callback = []
+ except Exception as e:
+ print(e)
+ raise e
+
+
# test_redis_cache_acompletion_stream_bedrock()
# redis cache with custom keys
def custom_get_cache_key(*args, **kwargs):
diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py
index 80a4372a572..1d0a74ff3ed 100644
--- a/litellm/tests/test_completion.py
+++ b/litellm/tests/test_completion.py
@@ -110,6 +110,32 @@ def test_completion_mistral_api():
# test_completion_mistral_api()
+def test_completion_mistral_api_modified_input():
+ try:
+ litellm.set_verbose = True
+ response = completion(
+ model="mistral/mistral-tiny",
+ max_tokens=5,
+ messages=[
+ {
+ "role": "user",
+ "content": [{"type": "text", "text": "Hey, how's it going?"}],
+ }
+ ],
+ )
+ # Add any assertions here to check the response
+ print(response)
+
+ cost = litellm.completion_cost(completion_response=response)
+ print("cost to make mistral completion=", cost)
+ assert cost > 0.0
+ except Exception as e:
+ if "500" in str(e):
+ pass
+ else:
+ pytest.fail(f"Error occurred: {e}")
+
+
def test_completion_claude2_1():
try:
print("claude2.1 test request")
@@ -1495,6 +1521,33 @@ def test_completion_chat_sagemaker_mistral():
# test_completion_chat_sagemaker_mistral()
+
+
+def test_completion_bedrock_titan_null_response():
+ try:
+ response = completion(
+ model="bedrock/amazon.titan-text-lite-v1",
+ messages=[
+ {
+ "role": "user",
+ "content": "Hello!",
+ },
+ {
+ "role": "assistant",
+ "content": "Hello! How can I help you?",
+ },
+ {
+ "role": "user",
+ "content": "What model are you?",
+ },
+ ],
+ )
+ # Add any assertions here to check the response
+ pytest.fail(f"Expected to fail")
+ except Exception as e:
+ pass
+
+
def test_completion_bedrock_titan():
try:
response = completion(
@@ -1743,7 +1796,7 @@ def test_azure_cloudflare_api():
def test_completion_anyscale_2():
try:
- # litellm.set_verbose=True
+ # litellm.set_verbose = True
messages = [
{"role": "system", "content": "You're a good bot"},
{
diff --git a/litellm/tests/test_custom_callback_router.py b/litellm/tests/test_custom_callback_router.py
index ac8b2fa1014..071d4529db7 100644
--- a/litellm/tests/test_custom_callback_router.py
+++ b/litellm/tests/test_custom_callback_router.py
@@ -256,6 +256,7 @@ class CompletionCustomHandler(
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
try:
self.states.append("async_success")
+ print("in async success, kwargs: ", kwargs)
## START TIME
assert isinstance(start_time, datetime)
## END TIME
@@ -266,6 +267,38 @@ class CompletionCustomHandler(
)
## KWARGS
assert isinstance(kwargs["model"], str)
+
+ # checking we use base_model for azure cost calculation
+ base_model = litellm.utils._get_base_model_from_metadata(
+ model_call_details=kwargs
+ )
+
+ if (
+ kwargs["model"] == "chatgpt-v-2"
+ and base_model is not None
+ and kwargs["stream"] != True
+ ):
+ # when base_model is set for azure, we should use pricing for the base_model
+ # this checks response_cost == litellm.cost_per_token(model=base_model)
+ assert isinstance(kwargs["response_cost"], float)
+ response_cost = kwargs["response_cost"]
+ print(
+ f"response_cost: {response_cost}, for model: {kwargs['model']} and base_model: {base_model}"
+ )
+ prompt_tokens = response_obj.usage.prompt_tokens
+ completion_tokens = response_obj.usage.completion_tokens
+ # ensure the pricing is based on the base_model here
+ prompt_price, completion_price = litellm.cost_per_token(
+ model=base_model,
+ prompt_tokens=prompt_tokens,
+ completion_tokens=completion_tokens,
+ )
+ expected_price = prompt_price + completion_price
+ print(f"expected price: {expected_price}")
+ assert (
+ response_cost == expected_price
+ ), f"response_cost: {response_cost} != expected_price: {expected_price}. For model: {kwargs['model']} and base_model: {base_model}. should have used base_model for price"
+
assert isinstance(kwargs["messages"], list)
assert isinstance(kwargs["optional_params"], dict)
assert isinstance(kwargs["litellm_params"], dict)
@@ -345,6 +378,7 @@ async def test_async_chat_azure():
customHandler_streaming_azure_router = CompletionCustomHandler()
customHandler_failure = CompletionCustomHandler()
litellm.callbacks = [customHandler_completion_azure_router]
+ litellm.set_verbose = True
model_list = [
{
"model_name": "gpt-3.5-turbo", # openai model name
@@ -354,6 +388,7 @@ async def test_async_chat_azure():
"api_version": os.getenv("AZURE_API_VERSION"),
"api_base": os.getenv("AZURE_API_BASE"),
},
+ "model_info": {"base_model": "azure/gpt-4-1106-preview"},
"tpm": 240000,
"rpm": 1800,
},
diff --git a/litellm/tests/test_load_test_router_s3.py b/litellm/tests/test_load_test_router_s3.py
new file mode 100644
index 00000000000..ed3df5f5d1a
--- /dev/null
+++ b/litellm/tests/test_load_test_router_s3.py
@@ -0,0 +1,94 @@
+# import sys, os
+# import traceback
+# from dotenv import load_dotenv
+# import copy
+
+# load_dotenv()
+# sys.path.insert(
+# 0, os.path.abspath("../..")
+# ) # Adds the parent directory to the system path
+# import asyncio
+# from litellm import Router, Timeout
+# import time
+# from litellm.caching import Cache
+# import litellm
+
+# litellm.cache = Cache(
+# type="s3", s3_bucket_name="cache-bucket-litellm", s3_region_name="us-west-2"
+# )
+
+# ### Test calling router with s3 Cache
+
+
+# async def call_acompletion(semaphore, router: Router, input_data):
+# async with semaphore:
+# try:
+# # Use asyncio.wait_for to set a timeout for the task
+# response = await router.acompletion(**input_data)
+# # Handle the response as needed
+# print(response)
+# return response
+# except Timeout:
+# print(f"Task timed out: {input_data}")
+# return None # You may choose to return something else or raise an exception
+
+
+# async def main():
+# # Initialize the Router
+# model_list = [
+# {
+# "model_name": "gpt-3.5-turbo",
+# "litellm_params": {
+# "model": "gpt-3.5-turbo",
+# "api_key": os.getenv("OPENAI_API_KEY"),
+# },
+# },
+# {
+# "model_name": "gpt-3.5-turbo",
+# "litellm_params": {
+# "model": "azure/chatgpt-v-2",
+# "api_key": os.getenv("AZURE_API_KEY"),
+# "api_base": os.getenv("AZURE_API_BASE"),
+# "api_version": os.getenv("AZURE_API_VERSION"),
+# },
+# },
+# ]
+# router = Router(model_list=model_list, num_retries=3, timeout=10)
+
+# # Create a semaphore with a capacity of 100
+# semaphore = asyncio.Semaphore(100)
+
+# # List to hold all task references
+# tasks = []
+# start_time_all_tasks = time.time()
+# # Launch 1000 tasks
+# for _ in range(500):
+# task = asyncio.create_task(
+# call_acompletion(
+# semaphore,
+# router,
+# {
+# "model": "gpt-3.5-turbo",
+# "messages": [{"role": "user", "content": "Hey, how's it going?"}],
+# },
+# )
+# )
+# tasks.append(task)
+
+# # Wait for all tasks to complete
+# responses = await asyncio.gather(*tasks)
+# # Process responses as needed
+# # Record the end time for all tasks
+# end_time_all_tasks = time.time()
+# # Calculate the total time for all tasks
+# total_time_all_tasks = end_time_all_tasks - start_time_all_tasks
+# print(f"Total time for all tasks: {total_time_all_tasks} seconds")
+
+# # Calculate the average time per response
+# average_time_per_response = total_time_all_tasks / len(responses)
+# print(f"Average time per response: {average_time_per_response} seconds")
+# print(f"NUMBER OF COMPLETED TASKS: {len(responses)}")
+
+
+# # Run the main function
+# asyncio.run(main())
diff --git a/litellm/tests/test_loadtest_router.py b/litellm/tests/test_loadtest_router.py
index fc3a7c5ad4b..a12a45b514f 100644
--- a/litellm/tests/test_loadtest_router.py
+++ b/litellm/tests/test_loadtest_router.py
@@ -9,6 +9,7 @@
# ) # Adds the parent directory to the system path
# import asyncio
# from litellm import Router, Timeout
+# import time
# async def call_acompletion(semaphore, router: Router, input_data):
@@ -17,6 +18,7 @@
# # Use asyncio.wait_for to set a timeout for the task
# response = await router.acompletion(**input_data)
# # Handle the response as needed
+# print(response)
# return response
# except Timeout:
# print(f"Task timed out: {input_data}")
@@ -42,15 +44,6 @@
# "api_version": os.getenv("AZURE_API_VERSION"),
# },
# },
-# {
-# "model_name": "gpt-3.5-turbo",
-# "litellm_params": {
-# "model": "azure/chatgpt-functioncalling",
-# "api_key": os.getenv("AZURE_API_KEY"),
-# "api_base": os.getenv("AZURE_API_BASE"),
-# "api_version": os.getenv("AZURE_API_VERSION"),
-# },
-# },
# ]
# router = Router(model_list=model_list, num_retries=3, timeout=10)
@@ -59,9 +52,9 @@
# # List to hold all task references
# tasks = []
-
+# start_time_all_tasks = time.time()
# # Launch 1000 tasks
-# for _ in range(1000):
+# for _ in range(500):
# task = asyncio.create_task(
# call_acompletion(
# semaphore,
@@ -77,6 +70,15 @@
# # Wait for all tasks to complete
# responses = await asyncio.gather(*tasks)
# # Process responses as needed
+# # Record the end time for all tasks
+# end_time_all_tasks = time.time()
+# # Calculate the total time for all tasks
+# total_time_all_tasks = end_time_all_tasks - start_time_all_tasks
+# print(f"Total time for all tasks: {total_time_all_tasks} seconds")
+
+# # Calculate the average time per response
+# average_time_per_response = total_time_all_tasks / len(responses)
+# print(f"Average time per response: {average_time_per_response} seconds")
# print(f"NUMBER OF COMPLETED TASKS: {len(responses)}")
diff --git a/litellm/tests/test_parallel_request_limiter.py b/litellm/tests/test_parallel_request_limiter.py
index bfac8ddeae9..17d79c36c9b 100644
--- a/litellm/tests/test_parallel_request_limiter.py
+++ b/litellm/tests/test_parallel_request_limiter.py
@@ -292,6 +292,7 @@ async def test_normal_router_call():
model="azure-model",
messages=[{"role": "user", "content": "Hey, how's it going?"}],
metadata={"user_api_key": _api_key},
+ mock_response="hello",
)
await asyncio.sleep(1) # success is done in a separate thread
print(f"response: {response}")
@@ -450,6 +451,7 @@ async def test_streaming_router_call():
messages=[{"role": "user", "content": "Hey, how's it going?"}],
stream=True,
metadata={"user_api_key": _api_key},
+ mock_response="hello",
)
async for chunk in response:
continue
@@ -526,6 +528,7 @@ async def test_streaming_router_tpm_limit():
messages=[{"role": "user", "content": "Write me a paragraph on the moon"}],
stream=True,
metadata={"user_api_key": _api_key},
+ mock_response="hello",
)
async for chunk in response:
continue
diff --git a/litellm/utils.py b/litellm/utils.py
index b37c68d6554..0ab5d672524 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -1079,10 +1079,17 @@ class Logging:
call_type=self.call_type,
)
else:
+ # check if base_model set on azure
+ base_model = _get_base_model_from_metadata(
+ model_call_details=self.model_call_details
+ )
+ # base_model defaults to None if not set on model_info
self.model_call_details[
"response_cost"
] = litellm.completion_cost(
- completion_response=result, call_type=self.call_type
+ completion_response=result,
+ call_type=self.call_type,
+ model=base_model,
)
verbose_logger.debug(
f"Model={self.model}; cost={self.model_call_details['response_cost']}"
@@ -1158,10 +1165,16 @@ class Logging:
if self.model_call_details.get("cache_hit", False) == True:
self.model_call_details["response_cost"] = 0.0
else:
+ # check if base_model set on azure
+ base_model = _get_base_model_from_metadata(
+ model_call_details=self.model_call_details
+ )
+ # base_model defaults to None if not set on model_info
self.model_call_details[
"response_cost"
] = litellm.completion_cost(
completion_response=complete_streaming_response,
+ model=base_model,
)
verbose_logger.debug(
f"Model={self.model}; cost={self.model_call_details['response_cost']}"
@@ -1369,6 +1382,36 @@ class Logging:
end_time=end_time,
print_verbose=print_verbose,
)
+ if callback == "s3":
+ global s3Logger
+ if s3Logger is None:
+ s3Logger = S3Logger()
+ if self.stream:
+ if "complete_streaming_response" in self.model_call_details:
+ print_verbose(
+ "S3Logger Logger: Got Stream Event - Completed Stream Response"
+ )
+ s3Logger.log_event(
+ kwargs=self.model_call_details,
+ response_obj=self.model_call_details[
+ "complete_streaming_response"
+ ],
+ start_time=start_time,
+ end_time=end_time,
+ print_verbose=print_verbose,
+ )
+ else:
+ print_verbose(
+ "S3Logger Logger: Got Stream Event - No complete stream response as yet"
+ )
+ else:
+ s3Logger.log_event(
+ kwargs=self.model_call_details,
+ response_obj=result,
+ start_time=start_time,
+ end_time=end_time,
+ print_verbose=print_verbose,
+ )
if (
isinstance(callback, CustomLogger)
and self.model_call_details.get("litellm_params", {}).get(
@@ -1479,8 +1522,14 @@ class Logging:
if self.model_call_details.get("cache_hit", False) == True:
self.model_call_details["response_cost"] = 0.0
else:
+ # check if base_model set on azure
+ base_model = _get_base_model_from_metadata(
+ model_call_details=self.model_call_details
+ )
+ # base_model defaults to None if not set on model_info
self.model_call_details["response_cost"] = litellm.completion_cost(
completion_response=complete_streaming_response,
+ model=base_model,
)
verbose_logger.debug(
f"Model={self.model}; cost={self.model_call_details['response_cost']}"
@@ -1527,7 +1576,7 @@ class Logging:
# only add to cache once we have a complete streaming response
litellm.cache.add_cache(result, **kwargs)
if isinstance(callback, CustomLogger): # custom logger class
- print_verbose(f"Async success callbacks: CustomLogger")
+ print_verbose(f"Async success callbacks: {callback}")
if self.stream:
if "complete_streaming_response" in self.model_call_details:
await callback.async_log_success_event(
@@ -1605,36 +1654,6 @@ class Logging:
end_time=end_time,
print_verbose=print_verbose,
)
- if callback == "s3":
- global s3Logger
- if s3Logger is None:
- s3Logger = S3Logger()
- if self.stream:
- if "complete_streaming_response" in self.model_call_details:
- print_verbose(
- "S3Logger Logger: Got Stream Event - Completed Stream Response"
- )
- await s3Logger._async_log_event(
- kwargs=self.model_call_details,
- response_obj=self.model_call_details[
- "complete_streaming_response"
- ],
- start_time=start_time,
- end_time=end_time,
- print_verbose=print_verbose,
- )
- else:
- print_verbose(
- "S3Logger Logger: Got Stream Event - No complete stream response as yet"
- )
- else:
- await s3Logger._async_log_event(
- kwargs=self.model_call_details,
- response_obj=result,
- start_time=start_time,
- end_time=end_time,
- print_verbose=print_verbose,
- )
except:
print_verbose(
f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while success logging {traceback.format_exc()}"
@@ -1990,11 +2009,6 @@ def client(original_function):
# we only support async dynamo db logging for acompletion/aembedding since that's used on proxy
litellm._async_success_callback.append(callback)
removed_async_items.append(index)
- elif callback == "s3":
- # s3 is an async callback, it's used for the proxy and needs to be async
- # we only support async s3 logging for acompletion/aembedding since that's used on proxy
- litellm._async_success_callback.append(callback)
- removed_async_items.append(index)
# Pop the async items from success_callback in reverse order to avoid index issues
for index in reversed(removed_async_items):
@@ -2767,6 +2781,12 @@ def client(original_function):
result, *args, **kwargs
)
)
+ elif isinstance(litellm.cache.cache, S3Cache):
+ threading.Thread(
+ target=litellm.cache.add_cache,
+ args=(result,) + args,
+ kwargs=kwargs,
+ ).start()
else:
asyncio.create_task(
litellm.cache.async_add_cache(
@@ -4358,7 +4378,7 @@ def get_optional_params(
if frequency_penalty is not None:
optional_params["repeat_penalty"] = frequency_penalty
if stop is not None:
- optional_params["stop_sequences"] = stop
+ optional_params["stop"] = stop
elif custom_llm_provider == "ollama_chat":
supported_params = [
"max_tokens",
@@ -4381,7 +4401,7 @@ def get_optional_params(
if frequency_penalty is not None:
optional_params["repeat_penalty"] = frequency_penalty
if stop is not None:
- optional_params["stop_sequences"] = stop
+ optional_params["stop"] = stop
elif custom_llm_provider == "nlp_cloud":
supported_params = [
"max_tokens",
@@ -8799,6 +8819,14 @@ def mock_completion_streaming_obj(model_response, mock_response, model):
yield model_response
+async def async_mock_completion_streaming_obj(model_response, mock_response, model):
+ for i in range(0, len(mock_response), 3):
+ completion_obj = Delta(role="assistant", content=mock_response)
+ model_response.choices[0].delta = completion_obj
+ model_response.choices[0].finish_reason = "stop"
+ yield model_response
+
+
########## Reading Config File ############################
def read_config_args(config_path) -> dict:
try:
@@ -9231,3 +9259,21 @@ def get_logging_id(start_time, response_obj):
return response_id
except:
return None
+
+
+def _get_base_model_from_metadata(model_call_details=None):
+ if model_call_details is None:
+ return None
+ litellm_params = model_call_details.get("litellm_params", {})
+
+ if litellm_params is not None:
+ metadata = litellm_params.get("metadata", {})
+
+ if metadata is not None:
+ model_info = metadata.get("model_info", {})
+
+ if model_info is not None:
+ base_model = model_info.get("base_model", None)
+ if base_model is not None:
+ return base_model
+ return None
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index 4c28bdbe8b3..e44f46ff84a 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -8,6 +8,15 @@
"litellm_provider": "openai",
"mode": "chat"
},
+ "gpt-4-turbo-preview": {
+ "max_tokens": 8192,
+ "max_input_tokens": 8192,
+ "max_output_tokens": 4096,
+ "input_cost_per_token": 0.00001,
+ "output_cost_per_token": 0.00003,
+ "litellm_provider": "openai",
+ "mode": "chat"
+ },
"gpt-4-0314": {
"max_tokens": 8192,
"max_input_tokens": 8192,
@@ -243,6 +252,15 @@
"output_cost_per_pixel": 0.0,
"litellm_provider": "openai"
},
+ "azure/gpt-4-0125-preview": {
+ "max_tokens": 128000,
+ "max_input_tokens": 128000,
+ "max_output_tokens": 4096,
+ "input_cost_per_token": 0.00001,
+ "output_cost_per_token": 0.00003,
+ "litellm_provider": "azure",
+ "mode": "chat"
+ },
"azure/gpt-4-1106-preview": {
"max_tokens": 128000,
"max_input_tokens": 128000,
diff --git a/pyproject.toml b/pyproject.toml
index 7f2062d5caa..fde1a27708f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm"
-version = "1.23.0"
+version = "1.23.5"
description = "Library to easily interface with LLM API providers"
authors = ["BerriAI"]
license = "MIT"
@@ -69,7 +69,7 @@ requires = ["poetry-core", "wheel"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
-version = "1.23.0"
+version = "1.23.5"
version_files = [
"pyproject.toml:^version"
]
diff --git a/schema.prisma b/schema.prisma
index 2164358ccc4..3a5c3ea230f 100644
--- a/schema.prisma
+++ b/schema.prisma
@@ -1,6 +1,7 @@
datasource client {
provider = "postgresql"
url = env("DATABASE_URL")
+ directUrl = env("DIRECT_URL")
}
generator client {
diff --git a/ui/litellm-dashboard/out/404.html b/ui/litellm-dashboard/out/404.html
index c57eb5193a2..74bcfb57bf0 100644
--- a/ui/litellm-dashboard/out/404.html
+++ b/ui/litellm-dashboard/out/404.html
@@ -1 +1 @@
-404
This page could not be found.
404
This page could not be found.
Loading...
\ No newline at end of file
+Loading...
\ No newline at end of file
diff --git a/ui/litellm-dashboard/out/index.txt b/ui/litellm-dashboard/out/index.txt
index f48954f2c46..97964f308ee 100644
--- a/ui/litellm-dashboard/out/index.txt
+++ b/ui/litellm-dashboard/out/index.txt
@@ -1,7 +1,7 @@
2:"$Sreact.suspense"
-3:I[27680,["787","static/chunks/787-5bb33960644f5c7c.js","931","static/chunks/app/page-7f03ccc8529ada97.js"],""]
+3:I[42989,["787","static/chunks/787-5bb33960644f5c7c.js","931","static/chunks/app/page-b1e8aae39e9306e9.js"],""]
4:I[5613,[],""]
5:I[31778,[],""]
-0:["p5gDwQBbgW8D3Uz3lgoZg",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},["$L1",["$","$2",null,{"fallback":["$","div",null,{"children":"Loading..."}],"children":["$","div",null,{"className":"flex min-h-screen flex-col ","children":["$","$L3",null,{}]}]}],null]]},[null,["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_c23dc8","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"loading":"$undefined","loadingStyles":"$undefined","loadingScripts":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[],"styles":null}]}]}],null]],[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/a6a9860a7fe022a9.css","precedence":"next","crossOrigin":""}]],"$L6"]]]]
+0:["Ul7bzKEbTPs-NwnRJpbmM",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},["$L1",["$","$2",null,{"fallback":["$","div",null,{"children":"Loading..."}],"children":["$","div",null,{"className":"flex min-h-screen flex-col ","children":["$","$L3",null,{}]}]}],null]]},[null,["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_c23dc8","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"loading":"$undefined","loadingStyles":"$undefined","loadingScripts":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[],"styles":null}]}]}],null]],[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/173c5a76ba57a3b8.css","precedence":"next","crossOrigin":""}]],"$L6"]]]]
6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"🚅 LiteLLM"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/ui/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","meta","5",{"name":"next-size-adjust"}]]
1:null
diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx
index 12eea2dd8ec..d7e77775683 100644
--- a/ui/litellm-dashboard/src/components/networking.tsx
+++ b/ui/litellm-dashboard/src/components/networking.tsx
@@ -162,3 +162,32 @@ export const keySpendLogsCall = async (accessToken: String, token: String) => {
throw error;
}
};
+
+
+
+export const spendUsersCall = async (accessToken: String, userID: String) => {
+ try {
+ const url = proxyBaseUrl ? `${proxyBaseUrl}/spend/users` : `/spend/users`;
+ console.log("in spendUsersCall:", url);
+ const response = await fetch(`${url}/?user_id=${userID}`, {
+ method: "GET",
+ headers: {
+ Authorization: `Bearer ${accessToken}`,
+ "Content-Type": "application/json",
+ },
+ });
+ if (!response.ok) {
+ const errorData = await response.text();
+ message.error(errorData);
+ throw new Error("Network response was not ok");
+ }
+
+ const data = await response.json();
+ console.log(data);
+ return data;
+ } catch (error) {
+ console.error("Failed to get spend for user", error);
+ throw error;
+ }
+};
+
diff --git a/ui/litellm-dashboard/src/components/user_dashboard.tsx b/ui/litellm-dashboard/src/components/user_dashboard.tsx
index b1a06939b16..6310de0e53c 100644
--- a/ui/litellm-dashboard/src/components/user_dashboard.tsx
+++ b/ui/litellm-dashboard/src/components/user_dashboard.tsx
@@ -120,7 +120,7 @@ const UserDashboard = () => {
RPM Limit: {item.rpm_limit? item.rpm_limit : "Unlimited"}
+
+ )}
);
};
diff --git a/ui/litellm-dashboard/src/components/view_user_spend.tsx b/ui/litellm-dashboard/src/components/view_user_spend.tsx
index 70431c9ae04..a2c61667d49 100644
--- a/ui/litellm-dashboard/src/components/view_user_spend.tsx
+++ b/ui/litellm-dashboard/src/components/view_user_spend.tsx
@@ -1,5 +1,5 @@
"use client";
-import React, { useEffect } from "react";
+import React, { useEffect, useState } from "react";
import { keyDeleteCall } from "./networking";
import { StatusOnlineIcon, TrashIcon } from "@heroicons/react/outline";
import { DonutChart } from "@tremor/react";
@@ -17,6 +17,7 @@ import {
Title,
Icon,
} from "@tremor/react";
+import { spendUsersCall } from "./networking";
// Define the props type
@@ -28,13 +29,33 @@ interface UserSpendData {
interface ViewUserSpendProps {
userID: string | null;
userSpendData: UserSpendData | null; // Use the UserSpendData interface here
+ userRole: string | null;
+ accessToken: string;
}
-const ViewUserSpend: React.FC
+
+
+ {/* Modal Panel */}
+
+
+ {/* Confirmation Modal Content */}
+
+
+
+
+
+
+
+
+
+ Delete Key
+
+
+ Are you sure you want to delete this key ?
+
+
+
+
+