mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-07 02:59:05 +00:00
Merge branch 'BerriAI:main' into ollama-image-handling
This commit is contained in:
frob
GitHub
commit
dd623d92c4
69 changed files with 2477 additions and 405 deletions
|
|
@ -57,6 +57,7 @@ jobs:
|
|||
pip install "pytest-mock==3.12.0"
|
||||
pip install python-multipart
|
||||
pip install google-cloud-aiplatform
|
||||
pip install prometheus-client==0.20.0
|
||||
- save_cache:
|
||||
paths:
|
||||
- ./venv
|
||||
|
|
@ -198,6 +199,10 @@ jobs:
|
|||
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
|
||||
-e AWS_REGION_NAME=$AWS_REGION_NAME \
|
||||
-e OPENAI_API_KEY=$OPENAI_API_KEY \
|
||||
-e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \
|
||||
-e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \
|
||||
-e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \
|
||||
-e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \
|
||||
--name my-app \
|
||||
-v $(pwd)/proxy_server_config.yaml:/app/config.yaml \
|
||||
my-app:latest \
|
||||
|
|
|
|||
|
|
@ -50,4 +50,105 @@ We support ALL Groq models, just set `groq/` as a prefix when sending completion
|
|||
|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| llama2-70b-4096 | `completion(model="groq/llama2-70b-4096", messages)` |
|
||||
| mixtral-8x7b-32768 | `completion(model="groq/mixtral-8x7b-32768", messages)` |
|
||||
| gemma-7b-it | `completion(model="groq/gemma-7b-it", messages)` |
|
||||
| gemma-7b-it | `completion(model="groq/gemma-7b-it", messages)` |
|
||||
|
||||
## Groq - Tool / Function Calling Example
|
||||
|
||||
```python
|
||||
# Example dummy function hard coded to return the current weather
|
||||
import json
|
||||
def get_current_weather(location, unit="fahrenheit"):
|
||||
"""Get the current weather in a given location"""
|
||||
if "tokyo" in location.lower():
|
||||
return json.dumps({"location": "Tokyo", "temperature": "10", "unit": "celsius"})
|
||||
elif "san francisco" in location.lower():
|
||||
return json.dumps(
|
||||
{"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"}
|
||||
)
|
||||
elif "paris" in location.lower():
|
||||
return json.dumps({"location": "Paris", "temperature": "22", "unit": "celsius"})
|
||||
else:
|
||||
return json.dumps({"location": location, "temperature": "unknown"})
|
||||
|
||||
|
||||
|
||||
|
||||
# Step 1: send the conversation and available functions to the model
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a function calling LLM that uses the data extracted from get_current_weather to answer questions about the weather in San Francisco.",
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's the weather like in San Francisco?",
|
||||
},
|
||||
]
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_current_weather",
|
||||
"description": "Get the current weather in a given location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {
|
||||
"type": "string",
|
||||
"description": "The city and state, e.g. San Francisco, CA",
|
||||
},
|
||||
"unit": {
|
||||
"type": "string",
|
||||
"enum": ["celsius", "fahrenheit"],
|
||||
},
|
||||
},
|
||||
"required": ["location"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
response = litellm.completion(
|
||||
model="groq/llama2-70b-4096",
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
tool_choice="auto", # auto is default, but we'll be explicit
|
||||
)
|
||||
print("Response\n", response)
|
||||
response_message = response.choices[0].message
|
||||
tool_calls = response_message.tool_calls
|
||||
|
||||
|
||||
# Step 2: check if the model wanted to call a function
|
||||
if tool_calls:
|
||||
# Step 3: call the function
|
||||
# Note: the JSON response may not always be valid; be sure to handle errors
|
||||
available_functions = {
|
||||
"get_current_weather": get_current_weather,
|
||||
}
|
||||
messages.append(
|
||||
response_message
|
||||
) # extend conversation with assistant's reply
|
||||
print("Response message\n", response_message)
|
||||
# Step 4: send the info for each function call and function response to the model
|
||||
for tool_call in tool_calls:
|
||||
function_name = tool_call.function.name
|
||||
function_to_call = available_functions[function_name]
|
||||
function_args = json.loads(tool_call.function.arguments)
|
||||
function_response = function_to_call(
|
||||
location=function_args.get("location"),
|
||||
unit=function_args.get("unit"),
|
||||
)
|
||||
messages.append(
|
||||
{
|
||||
"tool_call_id": tool_call.id,
|
||||
"role": "tool",
|
||||
"name": function_name,
|
||||
"content": function_response,
|
||||
}
|
||||
) # extend conversation with function response
|
||||
print(f"messages: {messages}")
|
||||
second_response = litellm.completion(
|
||||
model="groq/llama2-70b-4096", messages=messages
|
||||
) # get a new response from the model where it can see the function response
|
||||
print("second response\n", second_response)
|
||||
```
|
||||
|
|
@ -2,7 +2,7 @@ import Tabs from '@theme/Tabs';
|
|||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# OpenAI
|
||||
LiteLLM supports OpenAI Chat + Text completion and embedding calls.
|
||||
LiteLLM supports OpenAI Chat + Embedding calls.
|
||||
|
||||
### Required API Keys
|
||||
|
||||
|
|
@ -217,19 +217,6 @@ response = completion(
|
|||
|
||||
```
|
||||
|
||||
## OpenAI Text Completion Models / Instruct Models
|
||||
|
||||
| Model Name | Function Call |
|
||||
|---------------------|----------------------------------------------------|
|
||||
| gpt-3.5-turbo-instruct | `response = completion(model="gpt-3.5-turbo-instruct", messages=messages)` |
|
||||
| gpt-3.5-turbo-instruct-0914 | `response = completion(model="gpt-3.5-turbo-instruct-0914", messages=messages)` |
|
||||
| text-davinci-003 | `response = completion(model="text-davinci-003", messages=messages)` |
|
||||
| ada-001 | `response = completion(model="ada-001", messages=messages)` |
|
||||
| curie-001 | `response = completion(model="curie-001", messages=messages)` |
|
||||
| babbage-001 | `response = completion(model="babbage-001", messages=messages)` |
|
||||
| babbage-002 | `response = completion(model="babbage-002", messages=messages)` |
|
||||
| davinci-002 | `response = completion(model="davinci-002", messages=messages)` |
|
||||
|
||||
## Advanced
|
||||
|
||||
### Parallel Function calling
|
||||
|
|
|
|||
|
|
@ -5,7 +5,9 @@ import TabItem from '@theme/TabItem';
|
|||
|
||||
To call models hosted behind an openai proxy, make 2 changes:
|
||||
|
||||
1. Put `openai/` in front of your model name, so litellm knows you're trying to call an openai-compatible endpoint.
|
||||
1. For `/chat/completions`: Put `openai/` in front of your model name, so litellm knows you're trying to call an openai `/chat/completions` endpoint.
|
||||
|
||||
2. For `/completions`: Put `text-completion-openai/` in front of your model name, so litellm knows you're trying to call an openai `/completions` endpoint.
|
||||
|
||||
2. **Do NOT** add anything additional to the base url e.g. `/v1/embedding`. LiteLLM uses the openai-client to make these calls, and that automatically adds the relevant endpoints.
|
||||
|
||||
|
|
|
|||
163
docs/my-website/docs/providers/text_completion_openai.md
Normal file
163
docs/my-website/docs/providers/text_completion_openai.md
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
# OpenAI (Text Completion)
|
||||
|
||||
LiteLLM supports OpenAI text completion models
|
||||
|
||||
### Required API Keys
|
||||
|
||||
```python
|
||||
import os
|
||||
os.environ["OPENAI_API_KEY"] = "your-api-key"
|
||||
```
|
||||
|
||||
### Usage
|
||||
```python
|
||||
import os
|
||||
from litellm import completion
|
||||
|
||||
os.environ["OPENAI_API_KEY"] = "your-api-key"
|
||||
|
||||
# openai call
|
||||
response = completion(
|
||||
model = "gpt-3.5-turbo-instruct",
|
||||
messages=[{ "content": "Hello, how are you?","role": "user"}]
|
||||
)
|
||||
```
|
||||
|
||||
### Usage - LiteLLM Proxy Server
|
||||
|
||||
Here's how to call OpenAI models with the LiteLLM Proxy Server
|
||||
|
||||
### 1. Save key in your environment
|
||||
|
||||
```bash
|
||||
export OPENAI_API_KEY=""
|
||||
```
|
||||
|
||||
### 2. Start the proxy
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="config" label="config.yaml">
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: gpt-3.5-turbo
|
||||
litellm_params:
|
||||
model: openai/gpt-3.5-turbo # The `openai/` prefix will call openai.chat.completions.create
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
- model_name: gpt-3.5-turbo-instruct
|
||||
litellm_params:
|
||||
model: text-completion-openai/gpt-3.5-turbo-instruct # The `text-completion-openai/` prefix will call openai.completions.create
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
```
|
||||
</TabItem>
|
||||
<TabItem value="config-*" label="config.yaml - proxy all OpenAI models">
|
||||
|
||||
Use this to add all openai models with one API Key. **WARNING: This will not do any load balancing**
|
||||
This means requests to `gpt-4`, `gpt-3.5-turbo` , `gpt-4-turbo-preview` will all go through this route
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: "*" # all requests where model not in your config go to this deployment
|
||||
litellm_params:
|
||||
model: openai/* # set `openai/` to use the openai route
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
```
|
||||
</TabItem>
|
||||
<TabItem value="cli" label="CLI">
|
||||
|
||||
```bash
|
||||
$ litellm --model gpt-3.5-turbo-instruct
|
||||
|
||||
# Server running on http://0.0.0.0:4000
|
||||
```
|
||||
</TabItem>
|
||||
|
||||
</Tabs>
|
||||
|
||||
### 3. Test it
|
||||
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="Curl" label="Curl Request">
|
||||
|
||||
```shell
|
||||
curl --location 'http://0.0.0.0:4000/chat/completions' \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data ' {
|
||||
"model": "gpt-3.5-turbo-instruct",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "what llm are you"
|
||||
}
|
||||
]
|
||||
}
|
||||
'
|
||||
```
|
||||
</TabItem>
|
||||
<TabItem value="openai" label="OpenAI v1.0.0+">
|
||||
|
||||
```python
|
||||
import openai
|
||||
client = openai.OpenAI(
|
||||
api_key="anything",
|
||||
base_url="http://0.0.0.0:4000"
|
||||
)
|
||||
|
||||
# request sent to model set on litellm proxy, `litellm --model`
|
||||
response = client.chat.completions.create(model="gpt-3.5-turbo-instruct", messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "this is a test request, write a short poem"
|
||||
}
|
||||
])
|
||||
|
||||
print(response)
|
||||
|
||||
```
|
||||
</TabItem>
|
||||
<TabItem value="langchain" label="Langchain">
|
||||
|
||||
```python
|
||||
from langchain.chat_models import ChatOpenAI
|
||||
from langchain.prompts.chat import (
|
||||
ChatPromptTemplate,
|
||||
HumanMessagePromptTemplate,
|
||||
SystemMessagePromptTemplate,
|
||||
)
|
||||
from langchain.schema import HumanMessage, SystemMessage
|
||||
|
||||
chat = ChatOpenAI(
|
||||
openai_api_base="http://0.0.0.0:4000", # set openai_api_base to the LiteLLM Proxy
|
||||
model = "gpt-3.5-turbo-instruct",
|
||||
temperature=0.1
|
||||
)
|
||||
|
||||
messages = [
|
||||
SystemMessage(
|
||||
content="You are a helpful assistant that im using to make a test request to."
|
||||
),
|
||||
HumanMessage(
|
||||
content="test from litellm. tell me why it's amazing in 1 sentence"
|
||||
),
|
||||
]
|
||||
response = chat(messages)
|
||||
|
||||
print(response)
|
||||
```
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
|
||||
## OpenAI Text Completion Models / Instruct Models
|
||||
|
||||
| Model Name | Function Call |
|
||||
|---------------------|----------------------------------------------------|
|
||||
| gpt-3.5-turbo-instruct | `response = completion(model="gpt-3.5-turbo-instruct", messages=messages)` |
|
||||
| gpt-3.5-turbo-instruct-0914 | `response = completion(model="gpt-3.5-turbo-instruct-0914", messages=messages)` |
|
||||
| text-davinci-003 | `response = completion(model="text-davinci-003", messages=messages)` |
|
||||
| ada-001 | `response = completion(model="ada-001", messages=messages)` |
|
||||
| curie-001 | `response = completion(model="curie-001", messages=messages)` |
|
||||
| babbage-001 | `response = completion(model="babbage-001", messages=messages)` |
|
||||
| babbage-002 | `response = completion(model="babbage-002", messages=messages)` |
|
||||
| davinci-002 | `response = completion(model="davinci-002", messages=messages)` |
|
||||
|
|
@ -51,3 +51,21 @@ http://localhost:4000/metrics
|
|||
| `litellm_requests_metric` | Number of requests made, per `"user", "key", "model"` |
|
||||
| `litellm_spend_metric` | Total Spend, per `"user", "key", "model"` |
|
||||
| `litellm_total_tokens` | input + output tokens per `"user", "key", "model"` |
|
||||
|
||||
## Monitor System Health
|
||||
|
||||
To monitor the health of litellm adjacent services (redis / postgres), do:
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: gpt-3.5-turbo
|
||||
litellm_params:
|
||||
model: gpt-3.5-turbo
|
||||
litellm_settings:
|
||||
service_callback: ["prometheus_system"]
|
||||
```
|
||||
|
||||
| Metric Name | Description |
|
||||
|----------------------|--------------------------------------|
|
||||
| `litellm_redis_latency` | histogram latency for redis calls |
|
||||
| `litellm_redis_fails` | Number of failed redis calls |
|
||||
|
|
|
|||
|
|
@ -9,9 +9,9 @@ Log Proxy Input, Output, Exceptions using Custom Callbacks, Langfuse, OpenTeleme
|
|||
|
||||
- [Async Custom Callbacks](#custom-callback-class-async)
|
||||
- [Async Custom Callback APIs](#custom-callback-apis-async)
|
||||
- [Logging to DataDog](#logging-proxy-inputoutput---datadog)
|
||||
- [Logging to Langfuse](#logging-proxy-inputoutput---langfuse)
|
||||
- [Logging to s3 Buckets](#logging-proxy-inputoutput---s3-buckets)
|
||||
- [Logging to DataDog](#logging-proxy-inputoutput---datadog)
|
||||
- [Logging to DynamoDB](#logging-proxy-inputoutput---dynamodb)
|
||||
- [Logging to Sentry](#logging-proxy-inputoutput---sentry)
|
||||
- [Logging to Traceloop (OpenTelemetry)](#logging-proxy-inputoutput-traceloop-opentelemetry)
|
||||
|
|
@ -539,6 +539,36 @@ print(response)
|
|||
</Tabs>
|
||||
|
||||
|
||||
### Team based Logging to Langfuse
|
||||
|
||||
**Example:**
|
||||
|
||||
This config would send langfuse logs to 2 different langfuse projects, based on the team id
|
||||
|
||||
```yaml
|
||||
litellm_settings:
|
||||
default_team_settings:
|
||||
- team_id: my-secret-project
|
||||
success_callback: ["langfuse"]
|
||||
langfuse_public_key: os.environ/LANGFUSE_PUB_KEY_1 # Project 1
|
||||
langfuse_secret: os.environ/LANGFUSE_PRIVATE_KEY_1 # Project 1
|
||||
- team_id: ishaans-secret-project
|
||||
success_callback: ["langfuse"]
|
||||
langfuse_public_key: os.environ/LANGFUSE_PUB_KEY_2 # Project 2
|
||||
langfuse_secret: os.environ/LANGFUSE_SECRET_2 # Project 2
|
||||
```
|
||||
|
||||
Now, when you [generate keys](./virtual_keys.md) for this team-id
|
||||
|
||||
```bash
|
||||
curl -X POST 'http://0.0.0.0:4000/key/generate' \
|
||||
-H 'Authorization: Bearer sk-1234' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"team_id": "ishaans-secret-project"}'
|
||||
```
|
||||
|
||||
All requests made with these keys will log data to their team-specific logging.
|
||||
|
||||
## Logging Proxy Input/Output - DataDog
|
||||
We will use the `--config` to set `litellm.success_callback = ["datadog"]` this will log all successfull LLM calls to DataDog
|
||||
|
||||
|
|
|
|||
|
|
@ -115,6 +115,7 @@ const sidebars = {
|
|||
},
|
||||
items: [
|
||||
"providers/openai",
|
||||
"providers/text_completion_openai",
|
||||
"providers/openai_compatible",
|
||||
"providers/azure",
|
||||
"providers/azure_ai",
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ if set_verbose == True:
|
|||
input_callback: List[Union[str, Callable]] = []
|
||||
success_callback: List[Union[str, Callable]] = []
|
||||
failure_callback: List[Union[str, Callable]] = []
|
||||
service_callback: List[Union[str, Callable]] = []
|
||||
callbacks: List[Callable] = []
|
||||
_async_input_callback: List[Callable] = (
|
||||
[]
|
||||
|
|
|
|||
71
litellm/_service_logger.py
Normal file
71
litellm/_service_logger.py
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
import litellm
|
||||
from .types.services import ServiceTypes, ServiceLoggerPayload
|
||||
from .integrations.prometheus_services import PrometheusServicesLogger
|
||||
|
||||
|
||||
class ServiceLogging:
|
||||
"""
|
||||
Separate class used for monitoring health of litellm-adjacent services (redis/postgres).
|
||||
"""
|
||||
|
||||
def __init__(self, mock_testing: bool = False) -> None:
|
||||
self.mock_testing = mock_testing
|
||||
self.mock_testing_sync_success_hook = 0
|
||||
self.mock_testing_async_success_hook = 0
|
||||
self.mock_testing_sync_failure_hook = 0
|
||||
self.mock_testing_async_failure_hook = 0
|
||||
|
||||
if "prometheus_system" in litellm.service_callback:
|
||||
self.prometheusServicesLogger = PrometheusServicesLogger()
|
||||
|
||||
def service_success_hook(self, service: ServiceTypes, duration: float):
|
||||
"""
|
||||
[TODO] Not implemented for sync calls yet. V0 is focused on async monitoring (used by proxy).
|
||||
"""
|
||||
if self.mock_testing:
|
||||
self.mock_testing_sync_success_hook += 1
|
||||
|
||||
def service_failure_hook(
|
||||
self, service: ServiceTypes, duration: float, error: Exception
|
||||
):
|
||||
"""
|
||||
[TODO] Not implemented for sync calls yet. V0 is focused on async monitoring (used by proxy).
|
||||
"""
|
||||
if self.mock_testing:
|
||||
self.mock_testing_sync_failure_hook += 1
|
||||
|
||||
async def async_service_success_hook(self, service: ServiceTypes, duration: float):
|
||||
"""
|
||||
- For counting if the redis, postgres call is successful
|
||||
"""
|
||||
if self.mock_testing:
|
||||
self.mock_testing_async_success_hook += 1
|
||||
|
||||
payload = ServiceLoggerPayload(
|
||||
is_error=False, error=None, service=service, duration=duration
|
||||
)
|
||||
for callback in litellm.service_callback:
|
||||
if callback == "prometheus_system":
|
||||
await self.prometheusServicesLogger.async_service_success_hook(
|
||||
payload=payload
|
||||
)
|
||||
|
||||
async def async_service_failure_hook(
|
||||
self, service: ServiceTypes, duration: float, error: Exception
|
||||
):
|
||||
"""
|
||||
- For counting if the redis, postgres call is unsuccessful
|
||||
"""
|
||||
if self.mock_testing:
|
||||
self.mock_testing_async_failure_hook += 1
|
||||
|
||||
payload = ServiceLoggerPayload(
|
||||
is_error=True, error=str(error), service=service, duration=duration
|
||||
)
|
||||
for callback in litellm.service_callback:
|
||||
if callback == "prometheus_system":
|
||||
if self.prometheusServicesLogger is None:
|
||||
self.prometheusServicesLogger = self.prometheusServicesLogger()
|
||||
await self.prometheusServicesLogger.async_service_failure_hook(
|
||||
payload=payload
|
||||
)
|
||||
|
|
@ -13,6 +13,8 @@ import json, traceback, ast, hashlib
|
|||
from typing import Optional, Literal, List, Union, Any, BinaryIO
|
||||
from openai._models import BaseModel as OpenAIObject
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm._service_logger import ServiceLogging
|
||||
from litellm.types.services import ServiceLoggerPayload, ServiceTypes
|
||||
import traceback
|
||||
|
||||
|
||||
|
|
@ -163,6 +165,9 @@ class RedisCache(BaseCache):
|
|||
except Exception as e:
|
||||
pass
|
||||
|
||||
### HEALTH MONITORING OBJECT ###
|
||||
self.service_logger_obj = ServiceLogging()
|
||||
|
||||
def init_async_client(self):
|
||||
from ._redis import get_redis_async_client
|
||||
|
||||
|
|
@ -194,17 +199,59 @@ class RedisCache(BaseCache):
|
|||
)
|
||||
|
||||
async def async_scan_iter(self, pattern: str, count: int = 100) -> list:
|
||||
keys = []
|
||||
_redis_client = self.init_async_client()
|
||||
async with _redis_client as redis_client:
|
||||
async for key in redis_client.scan_iter(match=pattern + "*", count=count):
|
||||
keys.append(key)
|
||||
if len(keys) >= count:
|
||||
break
|
||||
return keys
|
||||
start_time = time.time()
|
||||
try:
|
||||
keys = []
|
||||
_redis_client = self.init_async_client()
|
||||
async with _redis_client as redis_client:
|
||||
async for key in redis_client.scan_iter(
|
||||
match=pattern + "*", count=count
|
||||
):
|
||||
keys.append(key)
|
||||
if len(keys) >= count:
|
||||
break
|
||||
|
||||
## LOGGING ##
|
||||
end_time = time.time()
|
||||
_duration = end_time - start_time
|
||||
asyncio.create_task(
|
||||
self.service_logger_obj.async_service_success_hook(
|
||||
service=ServiceTypes.REDIS, duration=_duration
|
||||
)
|
||||
) # DO NOT SLOW DOWN CALL B/C OF THIS
|
||||
return keys
|
||||
except Exception as e:
|
||||
# NON blocking - notify users Redis is throwing an exception
|
||||
## LOGGING ##
|
||||
end_time = time.time()
|
||||
_duration = end_time - start_time
|
||||
asyncio.create_task(
|
||||
self.service_logger_obj.async_service_failure_hook(
|
||||
service=ServiceTypes.REDIS, duration=_duration, error=e
|
||||
)
|
||||
)
|
||||
raise e
|
||||
|
||||
async def async_set_cache(self, key, value, **kwargs):
|
||||
_redis_client = self.init_async_client()
|
||||
start_time = time.time()
|
||||
try:
|
||||
_redis_client = self.init_async_client()
|
||||
except Exception as e:
|
||||
end_time = time.time()
|
||||
_duration = end_time - start_time
|
||||
asyncio.create_task(
|
||||
self.service_logger_obj.async_service_failure_hook(
|
||||
service=ServiceTypes.REDIS, duration=_duration, error=e
|
||||
)
|
||||
)
|
||||
# NON blocking - notify users Redis is throwing an exception
|
||||
verbose_logger.error(
|
||||
"LiteLLM Redis Caching: async set() - Got exception from REDIS %s, Writing value=%s",
|
||||
str(e),
|
||||
value,
|
||||
)
|
||||
traceback.print_exc()
|
||||
|
||||
key = self.check_and_fix_namespace(key=key)
|
||||
async with _redis_client as redis_client:
|
||||
ttl = kwargs.get("ttl", None)
|
||||
|
|
@ -216,7 +263,21 @@ class RedisCache(BaseCache):
|
|||
print_verbose(
|
||||
f"Successfully Set ASYNC Redis Cache: key: {key}\nValue {value}\nttl={ttl}"
|
||||
)
|
||||
end_time = time.time()
|
||||
_duration = end_time - start_time
|
||||
asyncio.create_task(
|
||||
self.service_logger_obj.async_service_success_hook(
|
||||
service=ServiceTypes.REDIS, duration=_duration
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
end_time = time.time()
|
||||
_duration = end_time - start_time
|
||||
asyncio.create_task(
|
||||
self.service_logger_obj.async_service_failure_hook(
|
||||
service=ServiceTypes.REDIS, duration=_duration, error=e
|
||||
)
|
||||
)
|
||||
# NON blocking - notify users Redis is throwing an exception
|
||||
verbose_logger.error(
|
||||
"LiteLLM Redis Caching: async set() - Got exception from REDIS %s, Writing value=%s",
|
||||
|
|
@ -230,6 +291,7 @@ class RedisCache(BaseCache):
|
|||
Use Redis Pipelines for bulk write operations
|
||||
"""
|
||||
_redis_client = self.init_async_client()
|
||||
start_time = time.time()
|
||||
try:
|
||||
async with _redis_client as redis_client:
|
||||
async with redis_client.pipeline(transaction=True) as pipe:
|
||||
|
|
@ -249,8 +311,25 @@ class RedisCache(BaseCache):
|
|||
|
||||
print_verbose(f"pipeline results: {results}")
|
||||
# Optionally, you could process 'results' to make sure that all set operations were successful.
|
||||
## LOGGING ##
|
||||
end_time = time.time()
|
||||
_duration = end_time - start_time
|
||||
asyncio.create_task(
|
||||
self.service_logger_obj.async_service_success_hook(
|
||||
service=ServiceTypes.REDIS, duration=_duration
|
||||
)
|
||||
)
|
||||
return results
|
||||
except Exception as e:
|
||||
## LOGGING ##
|
||||
end_time = time.time()
|
||||
_duration = end_time - start_time
|
||||
asyncio.create_task(
|
||||
self.service_logger_obj.async_service_failure_hook(
|
||||
service=ServiceTypes.REDIS, duration=_duration, error=e
|
||||
)
|
||||
)
|
||||
|
||||
verbose_logger.error(
|
||||
"LiteLLM Redis Caching: async set_cache_pipeline() - Got exception from REDIS %s, Writing value=%s",
|
||||
str(e),
|
||||
|
|
@ -265,15 +344,33 @@ class RedisCache(BaseCache):
|
|||
key = self.check_and_fix_namespace(key=key)
|
||||
self.redis_batch_writing_buffer.append((key, value))
|
||||
if len(self.redis_batch_writing_buffer) >= self.redis_flush_size:
|
||||
await self.flush_cache_buffer()
|
||||
await self.flush_cache_buffer() # logging done in here
|
||||
|
||||
async def async_increment(self, key, value: int, **kwargs) -> int:
|
||||
_redis_client = self.init_async_client()
|
||||
start_time = time.time()
|
||||
try:
|
||||
async with _redis_client as redis_client:
|
||||
result = await redis_client.incr(name=key, amount=value)
|
||||
## LOGGING ##
|
||||
end_time = time.time()
|
||||
_duration = end_time - start_time
|
||||
asyncio.create_task(
|
||||
self.service_logger_obj.async_service_success_hook(
|
||||
service=ServiceTypes.REDIS,
|
||||
duration=_duration,
|
||||
)
|
||||
)
|
||||
return result
|
||||
except Exception as e:
|
||||
## LOGGING ##
|
||||
end_time = time.time()
|
||||
_duration = end_time - start_time
|
||||
asyncio.create_task(
|
||||
self.service_logger_obj.async_service_failure_hook(
|
||||
service=ServiceTypes.REDIS, duration=_duration, error=e
|
||||
)
|
||||
)
|
||||
verbose_logger.error(
|
||||
"LiteLLM Redis Caching: async async_increment() - Got exception from REDIS %s, Writing value=%s",
|
||||
str(e),
|
||||
|
|
@ -348,6 +445,7 @@ class RedisCache(BaseCache):
|
|||
async def async_get_cache(self, key, **kwargs):
|
||||
_redis_client = self.init_async_client()
|
||||
key = self.check_and_fix_namespace(key=key)
|
||||
start_time = time.time()
|
||||
async with _redis_client as redis_client:
|
||||
try:
|
||||
print_verbose(f"Get Async Redis Cache: key: {key}")
|
||||
|
|
@ -356,8 +454,24 @@ class RedisCache(BaseCache):
|
|||
f"Got Async Redis Cache: key: {key}, cached_response {cached_response}"
|
||||
)
|
||||
response = self._get_cache_logic(cached_response=cached_response)
|
||||
## LOGGING ##
|
||||
end_time = time.time()
|
||||
_duration = end_time - start_time
|
||||
asyncio.create_task(
|
||||
self.service_logger_obj.async_service_success_hook(
|
||||
service=ServiceTypes.REDIS, duration=_duration
|
||||
)
|
||||
)
|
||||
return response
|
||||
except Exception as e:
|
||||
## LOGGING ##
|
||||
end_time = time.time()
|
||||
_duration = end_time - start_time
|
||||
asyncio.create_task(
|
||||
self.service_logger_obj.async_service_failure_hook(
|
||||
service=ServiceTypes.REDIS, duration=_duration, error=e
|
||||
)
|
||||
)
|
||||
# NON blocking - notify users Redis is throwing an exception
|
||||
print_verbose(
|
||||
f"LiteLLM Caching: async get() - Got exception from REDIS: {str(e)}"
|
||||
|
|
@ -369,6 +483,7 @@ class RedisCache(BaseCache):
|
|||
"""
|
||||
_redis_client = await self.init_async_client()
|
||||
key_value_dict = {}
|
||||
start_time = time.time()
|
||||
try:
|
||||
async with _redis_client as redis_client:
|
||||
_keys = []
|
||||
|
|
@ -377,17 +492,36 @@ class RedisCache(BaseCache):
|
|||
_keys.append(cache_key)
|
||||
results = await redis_client.mget(keys=_keys)
|
||||
|
||||
## LOGGING ##
|
||||
end_time = time.time()
|
||||
_duration = end_time - start_time
|
||||
asyncio.create_task(
|
||||
self.service_logger_obj.async_service_success_hook(
|
||||
service=ServiceTypes.REDIS, duration=_duration
|
||||
)
|
||||
)
|
||||
|
||||
# Associate the results back with their keys.
|
||||
# 'results' is a list of values corresponding to the order of keys in 'key_list'.
|
||||
key_value_dict = dict(zip(key_list, results))
|
||||
|
||||
decoded_results = {
|
||||
k.decode("utf-8"): self._get_cache_logic(v)
|
||||
for k, v in key_value_dict.items()
|
||||
}
|
||||
decoded_results = {}
|
||||
for k, v in key_value_dict.items():
|
||||
if isinstance(k, bytes):
|
||||
k = k.decode("utf-8")
|
||||
v = self._get_cache_logic(v)
|
||||
decoded_results[k] = v
|
||||
|
||||
return decoded_results
|
||||
except Exception as e:
|
||||
## LOGGING ##
|
||||
end_time = time.time()
|
||||
_duration = end_time - start_time
|
||||
asyncio.create_task(
|
||||
self.service_logger_obj.async_service_failure_hook(
|
||||
service=ServiceTypes.REDIS, duration=_duration, error=e
|
||||
)
|
||||
)
|
||||
print_verbose(f"Error occurred in pipeline read - {str(e)}")
|
||||
return key_value_dict
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# used for /metrics endpoint on LiteLLM Proxy
|
||||
#### What this does ####
|
||||
# On success + failure, log events to Supabase
|
||||
# On success, log events to Prometheus
|
||||
|
||||
import dotenv, os
|
||||
import requests
|
||||
|
|
|
|||
177
litellm/integrations/prometheus_services.py
Normal file
177
litellm/integrations/prometheus_services.py
Normal file
|
|
@ -0,0 +1,177 @@
|
|||
# used for monitoring litellm services health on `/metrics` endpoint on LiteLLM Proxy
|
||||
#### What this does ####
|
||||
# On success + failure, log events to Prometheus for litellm / adjacent services (litellm, redis, postgres, llm api providers)
|
||||
|
||||
|
||||
import dotenv, os
|
||||
import requests
|
||||
|
||||
dotenv.load_dotenv() # Loading env variables using dotenv
|
||||
import traceback
|
||||
import datetime, subprocess, sys
|
||||
import litellm, uuid
|
||||
from litellm._logging import print_verbose, verbose_logger
|
||||
from litellm.types.services import ServiceLoggerPayload, ServiceTypes
|
||||
|
||||
|
||||
class PrometheusServicesLogger:
|
||||
# Class variables or attributes
|
||||
litellm_service_latency = None # Class-level attribute to store the Histogram
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
mock_testing: bool = False,
|
||||
**kwargs,
|
||||
):
|
||||
try:
|
||||
try:
|
||||
from prometheus_client import Counter, Histogram, REGISTRY
|
||||
except ImportError:
|
||||
raise Exception(
|
||||
"Missing prometheus_client. Run `pip install prometheus-client`"
|
||||
)
|
||||
|
||||
self.Histogram = Histogram
|
||||
self.Counter = Counter
|
||||
self.REGISTRY = REGISTRY
|
||||
|
||||
verbose_logger.debug(f"in init prometheus services metrics")
|
||||
|
||||
self.services = [item.value for item in ServiceTypes]
|
||||
|
||||
self.payload_to_prometheus_map = (
|
||||
{}
|
||||
) # store the prometheus histogram/counter we need to call for each field in payload
|
||||
|
||||
for service in self.services:
|
||||
histogram = self.create_histogram(service)
|
||||
counter = self.create_counter(service)
|
||||
self.payload_to_prometheus_map[service] = [histogram, counter]
|
||||
|
||||
self.prometheus_to_amount_map: dict = (
|
||||
{}
|
||||
) # the field / value in ServiceLoggerPayload the object needs to be incremented by
|
||||
|
||||
### MOCK TESTING ###
|
||||
self.mock_testing = mock_testing
|
||||
self.mock_testing_success_calls = 0
|
||||
self.mock_testing_failure_calls = 0
|
||||
|
||||
except Exception as e:
|
||||
print_verbose(f"Got exception on init prometheus client {str(e)}")
|
||||
raise e
|
||||
|
||||
def is_metric_registered(self, metric_name) -> bool:
|
||||
for metric in self.REGISTRY.collect():
|
||||
if metric_name == metric.name:
|
||||
return True
|
||||
return False
|
||||
|
||||
def get_metric(self, metric_name):
|
||||
for metric in self.REGISTRY.collect():
|
||||
for sample in metric.samples:
|
||||
if metric_name == sample.name:
|
||||
return metric
|
||||
return None
|
||||
|
||||
def create_histogram(self, label: str):
|
||||
metric_name = "litellm_{}_latency".format(label)
|
||||
is_registered = self.is_metric_registered(metric_name)
|
||||
if is_registered:
|
||||
return self.get_metric(metric_name)
|
||||
return self.Histogram(
|
||||
metric_name,
|
||||
"Latency for {} service".format(label),
|
||||
labelnames=[label],
|
||||
)
|
||||
|
||||
def create_counter(self, label: str):
|
||||
metric_name = "litellm_{}_failed_requests".format(label)
|
||||
is_registered = self.is_metric_registered(metric_name)
|
||||
if is_registered:
|
||||
return self.get_metric(metric_name)
|
||||
return self.Counter(
|
||||
metric_name,
|
||||
"Total failed requests for {} service".format(label),
|
||||
labelnames=[label],
|
||||
)
|
||||
|
||||
def observe_histogram(
|
||||
self,
|
||||
histogram,
|
||||
labels: str,
|
||||
amount: float,
|
||||
):
|
||||
assert isinstance(histogram, self.Histogram)
|
||||
|
||||
histogram.labels(labels).observe(amount)
|
||||
|
||||
def increment_counter(
|
||||
self,
|
||||
counter,
|
||||
labels: str,
|
||||
amount: float,
|
||||
):
|
||||
assert isinstance(counter, self.Counter)
|
||||
|
||||
counter.labels(labels).inc(amount)
|
||||
|
||||
def service_success_hook(self, payload: ServiceLoggerPayload):
|
||||
if self.mock_testing:
|
||||
self.mock_testing_success_calls += 1
|
||||
|
||||
if payload.service.value in self.payload_to_prometheus_map:
|
||||
prom_objects = self.payload_to_prometheus_map[payload.service.value]
|
||||
for obj in prom_objects:
|
||||
if isinstance(obj, self.Histogram):
|
||||
self.observe_histogram(
|
||||
histogram=obj,
|
||||
labels=payload.service.value,
|
||||
amount=payload.duration,
|
||||
)
|
||||
|
||||
def service_failure_hook(self, payload: ServiceLoggerPayload):
|
||||
if self.mock_testing:
|
||||
self.mock_testing_failure_calls += 1
|
||||
|
||||
if payload.service.value in self.payload_to_prometheus_map:
|
||||
prom_objects = self.payload_to_prometheus_map[payload.service.value]
|
||||
for obj in prom_objects:
|
||||
if isinstance(obj, self.Counter):
|
||||
self.increment_counter(
|
||||
counter=obj,
|
||||
labels=payload.service.value,
|
||||
amount=1, # LOG ERROR COUNT TO PROMETHEUS
|
||||
)
|
||||
|
||||
async def async_service_success_hook(self, payload: ServiceLoggerPayload):
|
||||
"""
|
||||
Log successful call to prometheus
|
||||
"""
|
||||
if self.mock_testing:
|
||||
self.mock_testing_success_calls += 1
|
||||
|
||||
if payload.service.value in self.payload_to_prometheus_map:
|
||||
prom_objects = self.payload_to_prometheus_map[payload.service.value]
|
||||
for obj in prom_objects:
|
||||
if isinstance(obj, self.Histogram):
|
||||
self.observe_histogram(
|
||||
histogram=obj,
|
||||
labels=payload.service.value,
|
||||
amount=payload.duration,
|
||||
)
|
||||
|
||||
async def async_service_failure_hook(self, payload: ServiceLoggerPayload):
|
||||
print(f"received error payload: {payload.error}")
|
||||
if self.mock_testing:
|
||||
self.mock_testing_failure_calls += 1
|
||||
|
||||
if payload.service.value in self.payload_to_prometheus_map:
|
||||
prom_objects = self.payload_to_prometheus_map[payload.service.value]
|
||||
for obj in prom_objects:
|
||||
if isinstance(obj, self.Counter):
|
||||
self.increment_counter(
|
||||
counter=obj,
|
||||
labels=payload.service.value,
|
||||
amount=1, # LOG ERROR COUNT TO PROMETHEUS
|
||||
)
|
||||
|
|
@ -8,6 +8,8 @@ from litellm.utils import ModelResponse, Usage, CustomStreamWrapper
|
|||
import litellm
|
||||
from .prompt_templates.factory import prompt_factory, custom_prompt
|
||||
import httpx
|
||||
from .base import BaseLLM
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
|
||||
|
||||
class AnthropicConstants(Enum):
|
||||
|
|
@ -94,98 +96,13 @@ def validate_environment(api_key, user_headers):
|
|||
return headers
|
||||
|
||||
|
||||
def completion(
|
||||
model: str,
|
||||
messages: list,
|
||||
api_base: str,
|
||||
custom_prompt_dict: dict,
|
||||
model_response: ModelResponse,
|
||||
print_verbose: Callable,
|
||||
encoding,
|
||||
api_key,
|
||||
logging_obj,
|
||||
optional_params=None,
|
||||
litellm_params=None,
|
||||
logger_fn=None,
|
||||
headers={},
|
||||
):
|
||||
headers = validate_environment(api_key, headers)
|
||||
if model in custom_prompt_dict:
|
||||
# check if the model has a registered custom prompt
|
||||
model_prompt_details = custom_prompt_dict[model]
|
||||
prompt = custom_prompt(
|
||||
role_dict=model_prompt_details["roles"],
|
||||
initial_prompt_value=model_prompt_details["initial_prompt_value"],
|
||||
final_prompt_value=model_prompt_details["final_prompt_value"],
|
||||
messages=messages,
|
||||
)
|
||||
else:
|
||||
prompt = prompt_factory(
|
||||
model=model, messages=messages, custom_llm_provider="anthropic"
|
||||
)
|
||||
class AnthropicTextCompletion(BaseLLM):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
|
||||
## Load Config
|
||||
config = litellm.AnthropicTextConfig.get_config()
|
||||
for k, v in config.items():
|
||||
if (
|
||||
k not in optional_params
|
||||
): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in
|
||||
optional_params[k] = v
|
||||
|
||||
data = {
|
||||
"model": model,
|
||||
"prompt": prompt,
|
||||
**optional_params,
|
||||
}
|
||||
|
||||
## LOGGING
|
||||
logging_obj.pre_call(
|
||||
input=prompt,
|
||||
api_key=api_key,
|
||||
additional_args={
|
||||
"complete_input_dict": data,
|
||||
"api_base": api_base,
|
||||
"headers": headers,
|
||||
},
|
||||
)
|
||||
|
||||
## COMPLETION CALL
|
||||
if "stream" in optional_params and optional_params["stream"] == True:
|
||||
response = requests.post(
|
||||
api_base,
|
||||
headers=headers,
|
||||
data=json.dumps(data),
|
||||
stream=optional_params["stream"],
|
||||
)
|
||||
|
||||
if response.status_code != 200:
|
||||
raise AnthropicError(
|
||||
status_code=response.status_code, message=response.text
|
||||
)
|
||||
completion_stream = response.iter_lines()
|
||||
stream_response = CustomStreamWrapper(
|
||||
completion_stream=completion_stream,
|
||||
model=model,
|
||||
custom_llm_provider="anthropic",
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
return stream_response
|
||||
|
||||
else:
|
||||
response = requests.post(api_base, headers=headers, data=json.dumps(data))
|
||||
if response.status_code != 200:
|
||||
raise AnthropicError(
|
||||
status_code=response.status_code, message=response.text
|
||||
)
|
||||
|
||||
## LOGGING
|
||||
logging_obj.post_call(
|
||||
input=prompt,
|
||||
api_key=api_key,
|
||||
original_response=response.text,
|
||||
additional_args={"complete_input_dict": data},
|
||||
)
|
||||
print_verbose(f"raw model_response: {response.text}")
|
||||
def process_response(
|
||||
self, model_response: ModelResponse, response, encoding, prompt: str, model: str
|
||||
):
|
||||
## RESPONSE OBJECT
|
||||
try:
|
||||
completion_response = response.json()
|
||||
|
|
@ -221,9 +138,206 @@ def completion(
|
|||
total_tokens=prompt_tokens + completion_tokens,
|
||||
)
|
||||
model_response.usage = usage
|
||||
|
||||
return model_response
|
||||
|
||||
async def async_completion(
|
||||
self,
|
||||
model: str,
|
||||
model_response: ModelResponse,
|
||||
api_base: str,
|
||||
logging_obj,
|
||||
encoding,
|
||||
headers: dict,
|
||||
data: dict,
|
||||
client=None,
|
||||
):
|
||||
if client is None:
|
||||
client = AsyncHTTPHandler(timeout=httpx.Timeout(timeout=600.0, connect=5.0))
|
||||
|
||||
def embedding():
|
||||
# logic for parsing in - calling - parsing out model embedding calls
|
||||
pass
|
||||
response = await client.post(api_base, headers=headers, data=json.dumps(data))
|
||||
|
||||
if response.status_code != 200:
|
||||
raise AnthropicError(
|
||||
status_code=response.status_code, message=response.text
|
||||
)
|
||||
|
||||
## LOGGING
|
||||
logging_obj.post_call(
|
||||
input=data["prompt"],
|
||||
api_key=headers.get("x-api-key"),
|
||||
original_response=response.text,
|
||||
additional_args={"complete_input_dict": data},
|
||||
)
|
||||
|
||||
response = self.process_response(
|
||||
model_response=model_response,
|
||||
response=response,
|
||||
encoding=encoding,
|
||||
prompt=data["prompt"],
|
||||
model=model,
|
||||
)
|
||||
return response
|
||||
|
||||
async def async_streaming(
|
||||
self,
|
||||
model: str,
|
||||
api_base: str,
|
||||
logging_obj,
|
||||
headers: dict,
|
||||
data: Optional[dict],
|
||||
client=None,
|
||||
):
|
||||
if client is None:
|
||||
client = AsyncHTTPHandler(timeout=httpx.Timeout(timeout=600.0, connect=5.0))
|
||||
|
||||
response = await client.post(api_base, headers=headers, data=json.dumps(data))
|
||||
|
||||
if response.status_code != 200:
|
||||
raise AnthropicError(
|
||||
status_code=response.status_code, message=response.text
|
||||
)
|
||||
|
||||
completion_stream = response.aiter_lines()
|
||||
|
||||
streamwrapper = CustomStreamWrapper(
|
||||
completion_stream=completion_stream,
|
||||
model=model,
|
||||
custom_llm_provider="anthropic_text",
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
return streamwrapper
|
||||
|
||||
def completion(
|
||||
self,
|
||||
model: str,
|
||||
messages: list,
|
||||
api_base: str,
|
||||
acompletion: str,
|
||||
custom_prompt_dict: dict,
|
||||
model_response: ModelResponse,
|
||||
print_verbose: Callable,
|
||||
encoding,
|
||||
api_key,
|
||||
logging_obj,
|
||||
optional_params=None,
|
||||
litellm_params=None,
|
||||
logger_fn=None,
|
||||
headers={},
|
||||
client=None,
|
||||
):
|
||||
headers = validate_environment(api_key, headers)
|
||||
if model in custom_prompt_dict:
|
||||
# check if the model has a registered custom prompt
|
||||
model_prompt_details = custom_prompt_dict[model]
|
||||
prompt = custom_prompt(
|
||||
role_dict=model_prompt_details["roles"],
|
||||
initial_prompt_value=model_prompt_details["initial_prompt_value"],
|
||||
final_prompt_value=model_prompt_details["final_prompt_value"],
|
||||
messages=messages,
|
||||
)
|
||||
else:
|
||||
prompt = prompt_factory(
|
||||
model=model, messages=messages, custom_llm_provider="anthropic"
|
||||
)
|
||||
|
||||
## Load Config
|
||||
config = litellm.AnthropicTextConfig.get_config()
|
||||
for k, v in config.items():
|
||||
if (
|
||||
k not in optional_params
|
||||
): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in
|
||||
optional_params[k] = v
|
||||
|
||||
data = {
|
||||
"model": model,
|
||||
"prompt": prompt,
|
||||
**optional_params,
|
||||
}
|
||||
|
||||
## LOGGING
|
||||
logging_obj.pre_call(
|
||||
input=prompt,
|
||||
api_key=api_key,
|
||||
additional_args={
|
||||
"complete_input_dict": data,
|
||||
"api_base": api_base,
|
||||
"headers": headers,
|
||||
},
|
||||
)
|
||||
|
||||
## COMPLETION CALL
|
||||
if "stream" in optional_params and optional_params["stream"] == True:
|
||||
if acompletion == True:
|
||||
return self.async_streaming(
|
||||
model=model,
|
||||
api_base=api_base,
|
||||
logging_obj=logging_obj,
|
||||
headers=headers,
|
||||
data=data,
|
||||
client=None,
|
||||
)
|
||||
|
||||
if client is None:
|
||||
client = HTTPHandler(timeout=httpx.Timeout(timeout=600.0, connect=5.0))
|
||||
|
||||
response = client.post(
|
||||
api_base,
|
||||
headers=headers,
|
||||
data=json.dumps(data),
|
||||
# stream=optional_params["stream"],
|
||||
)
|
||||
|
||||
if response.status_code != 200:
|
||||
raise AnthropicError(
|
||||
status_code=response.status_code, message=response.text
|
||||
)
|
||||
completion_stream = response.iter_lines()
|
||||
stream_response = CustomStreamWrapper(
|
||||
completion_stream=completion_stream,
|
||||
model=model,
|
||||
custom_llm_provider="anthropic_text",
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
return stream_response
|
||||
elif acompletion == True:
|
||||
return self.async_completion(
|
||||
model=model,
|
||||
model_response=model_response,
|
||||
api_base=api_base,
|
||||
logging_obj=logging_obj,
|
||||
encoding=encoding,
|
||||
headers=headers,
|
||||
data=data,
|
||||
client=client,
|
||||
)
|
||||
else:
|
||||
if client is None:
|
||||
client = HTTPHandler(timeout=httpx.Timeout(timeout=600.0, connect=5.0))
|
||||
response = client.post(api_base, headers=headers, data=json.dumps(data))
|
||||
if response.status_code != 200:
|
||||
raise AnthropicError(
|
||||
status_code=response.status_code, message=response.text
|
||||
)
|
||||
|
||||
## LOGGING
|
||||
logging_obj.post_call(
|
||||
input=prompt,
|
||||
api_key=api_key,
|
||||
original_response=response.text,
|
||||
additional_args={"complete_input_dict": data},
|
||||
)
|
||||
print_verbose(f"raw model_response: {response.text}")
|
||||
|
||||
response = self.process_response(
|
||||
model_response=model_response,
|
||||
response=response,
|
||||
encoding=encoding,
|
||||
prompt=data["prompt"],
|
||||
model=model,
|
||||
)
|
||||
return response
|
||||
|
||||
def embedding(self):
|
||||
# logic for parsing in - calling - parsing out model embedding calls
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -58,13 +58,16 @@ class AsyncHTTPHandler:
|
|||
|
||||
|
||||
class HTTPHandler:
|
||||
def __init__(self, concurrent_limit=1000):
|
||||
def __init__(
|
||||
self, timeout: httpx.Timeout = _DEFAULT_TIMEOUT, concurrent_limit=1000
|
||||
):
|
||||
# Create a client with a connection pool
|
||||
self.client = httpx.Client(
|
||||
timeout=timeout,
|
||||
limits=httpx.Limits(
|
||||
max_connections=concurrent_limit,
|
||||
max_keepalive_connections=concurrent_limit,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
def close(self):
|
||||
|
|
|
|||
|
|
@ -270,6 +270,7 @@ def completion(
|
|||
logging_obj,
|
||||
vertex_project=None,
|
||||
vertex_location=None,
|
||||
vertex_credentials=None,
|
||||
optional_params=None,
|
||||
litellm_params=None,
|
||||
logger_fn=None,
|
||||
|
|
@ -348,6 +349,7 @@ def completion(
|
|||
print_verbose(
|
||||
f"VERTEX AI: vertex_project={vertex_project}; vertex_location={vertex_location}"
|
||||
)
|
||||
|
||||
creds, _ = google.auth.default(quota_project_id=vertex_project)
|
||||
print_verbose(
|
||||
f"VERTEX AI: creds={creds}; google application credentials: {os.getenv('GOOGLE_APPLICATION_CREDENTIALS')}"
|
||||
|
|
|
|||
|
|
@ -129,6 +129,18 @@ class VertexAIAnthropicConfig:
|
|||
|
||||
|
||||
# makes headers for API call
|
||||
def refresh_auth(
|
||||
credentials,
|
||||
) -> str: # used when user passes in credentials as json string
|
||||
from google.auth.transport.requests import Request # type: ignore[import-untyped]
|
||||
|
||||
if credentials.token is None:
|
||||
credentials.refresh(Request())
|
||||
|
||||
if not credentials.token:
|
||||
raise RuntimeError("Could not resolve API token from the credentials")
|
||||
|
||||
return credentials.token
|
||||
|
||||
|
||||
def completion(
|
||||
|
|
@ -140,6 +152,7 @@ def completion(
|
|||
logging_obj,
|
||||
vertex_project=None,
|
||||
vertex_location=None,
|
||||
vertex_credentials=None,
|
||||
optional_params=None,
|
||||
litellm_params=None,
|
||||
logger_fn=None,
|
||||
|
|
@ -217,11 +230,28 @@ def completion(
|
|||
## Completion Call
|
||||
|
||||
print_verbose(
|
||||
f"VERTEX AI: vertex_project={vertex_project}; vertex_location={vertex_location}"
|
||||
f"VERTEX AI: vertex_project={vertex_project}; vertex_location={vertex_location}; vertex_credentials={vertex_credentials}"
|
||||
)
|
||||
access_token = None
|
||||
if client is None:
|
||||
if vertex_credentials is not None and isinstance(vertex_credentials, str):
|
||||
import google.oauth2.service_account
|
||||
|
||||
json_obj = json.loads(vertex_credentials)
|
||||
|
||||
creds = (
|
||||
google.oauth2.service_account.Credentials.from_service_account_info(
|
||||
json_obj,
|
||||
scopes=["https://www.googleapis.com/auth/cloud-platform"],
|
||||
)
|
||||
)
|
||||
### CHECK IF ACCESS
|
||||
access_token = refresh_auth(credentials=creds)
|
||||
|
||||
vertex_ai_client = AnthropicVertex(
|
||||
project_id=vertex_project, region=vertex_location
|
||||
project_id=vertex_project,
|
||||
region=vertex_location,
|
||||
access_token=access_token,
|
||||
)
|
||||
else:
|
||||
vertex_ai_client = client
|
||||
|
|
@ -243,6 +273,7 @@ def completion(
|
|||
vertex_location=vertex_location,
|
||||
optional_params=optional_params,
|
||||
client=client,
|
||||
access_token=access_token,
|
||||
)
|
||||
else:
|
||||
return async_completion(
|
||||
|
|
@ -256,6 +287,7 @@ def completion(
|
|||
vertex_location=vertex_location,
|
||||
optional_params=optional_params,
|
||||
client=client,
|
||||
access_token=access_token,
|
||||
)
|
||||
if stream is not None and stream == True:
|
||||
## LOGGING
|
||||
|
|
@ -334,12 +366,13 @@ async def async_completion(
|
|||
vertex_location=None,
|
||||
optional_params=None,
|
||||
client=None,
|
||||
access_token=None,
|
||||
):
|
||||
from anthropic import AsyncAnthropicVertex
|
||||
|
||||
if client is None:
|
||||
vertex_ai_client = AsyncAnthropicVertex(
|
||||
project_id=vertex_project, region=vertex_location
|
||||
project_id=vertex_project, region=vertex_location, access_token=access_token
|
||||
)
|
||||
else:
|
||||
vertex_ai_client = client
|
||||
|
|
@ -404,12 +437,13 @@ async def async_streaming(
|
|||
vertex_location=None,
|
||||
optional_params=None,
|
||||
client=None,
|
||||
access_token=None,
|
||||
):
|
||||
from anthropic import AsyncAnthropicVertex
|
||||
|
||||
if client is None:
|
||||
vertex_ai_client = AsyncAnthropicVertex(
|
||||
project_id=vertex_project, region=vertex_location
|
||||
project_id=vertex_project, region=vertex_location, access_token=access_token
|
||||
)
|
||||
else:
|
||||
vertex_ai_client = client
|
||||
|
|
|
|||
|
|
@ -12,7 +12,6 @@ from typing import Any, Literal, Union, BinaryIO
|
|||
from functools import partial
|
||||
import dotenv, traceback, random, asyncio, time, contextvars
|
||||
from copy import deepcopy
|
||||
|
||||
import httpx
|
||||
import litellm
|
||||
from ._logging import verbose_logger
|
||||
|
|
@ -68,6 +67,7 @@ from .llms.openai import OpenAIChatCompletion, OpenAITextCompletion
|
|||
from .llms.azure import AzureChatCompletion
|
||||
from .llms.azure_text import AzureTextCompletion
|
||||
from .llms.anthropic import AnthropicChatCompletion
|
||||
from .llms.anthropic_text import AnthropicTextCompletion
|
||||
from .llms.huggingface_restapi import Huggingface
|
||||
from .llms.prompt_templates.factory import (
|
||||
prompt_factory,
|
||||
|
|
@ -100,6 +100,7 @@ dotenv.load_dotenv() # Loading env variables using dotenv
|
|||
openai_chat_completions = OpenAIChatCompletion()
|
||||
openai_text_completions = OpenAITextCompletion()
|
||||
anthropic_chat_completions = AnthropicChatCompletion()
|
||||
anthropic_text_completions = AnthropicTextCompletion()
|
||||
azure_chat_completions = AzureChatCompletion()
|
||||
azure_text_completions = AzureTextCompletion()
|
||||
huggingface = Huggingface()
|
||||
|
|
@ -1166,10 +1167,11 @@ def completion(
|
|||
or get_secret("ANTHROPIC_API_BASE")
|
||||
or "https://api.anthropic.com/v1/complete"
|
||||
)
|
||||
response = anthropic_text.completion(
|
||||
response = anthropic_text_completions.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
custom_prompt_dict=litellm.custom_prompt_dict,
|
||||
model_response=model_response,
|
||||
print_verbose=print_verbose,
|
||||
|
|
@ -1674,7 +1676,11 @@ def completion(
|
|||
or litellm.vertex_location
|
||||
or get_secret("VERTEXAI_LOCATION")
|
||||
)
|
||||
|
||||
vertex_credentials = (
|
||||
optional_params.pop("vertex_credentials", None)
|
||||
or optional_params.pop("vertex_ai_credentials", None)
|
||||
or get_secret("VERTEXAI_CREDENTIALS")
|
||||
)
|
||||
if "claude-3" in model:
|
||||
model_response = vertex_ai_anthropic.completion(
|
||||
model=model,
|
||||
|
|
@ -1687,6 +1693,7 @@ def completion(
|
|||
encoding=encoding,
|
||||
vertex_location=vertex_ai_location,
|
||||
vertex_project=vertex_ai_project,
|
||||
vertex_credentials=vertex_credentials,
|
||||
logging_obj=logging,
|
||||
acompletion=acompletion,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -714,7 +714,8 @@
|
|||
"input_cost_per_token": 0.00000070,
|
||||
"output_cost_per_token": 0.00000080,
|
||||
"litellm_provider": "groq",
|
||||
"mode": "chat"
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true
|
||||
},
|
||||
"groq/mixtral-8x7b-32768": {
|
||||
"max_tokens": 32768,
|
||||
|
|
@ -723,7 +724,8 @@
|
|||
"input_cost_per_token": 0.00000027,
|
||||
"output_cost_per_token": 0.00000027,
|
||||
"litellm_provider": "groq",
|
||||
"mode": "chat"
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true
|
||||
},
|
||||
"groq/gemma-7b-it": {
|
||||
"max_tokens": 8192,
|
||||
|
|
@ -732,7 +734,8 @@
|
|||
"input_cost_per_token": 0.00000010,
|
||||
"output_cost_per_token": 0.00000010,
|
||||
"litellm_provider": "groq",
|
||||
"mode": "chat"
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true
|
||||
},
|
||||
"claude-instant-1.2": {
|
||||
"max_tokens": 8191,
|
||||
|
|
@ -1079,7 +1082,8 @@
|
|||
"source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models"
|
||||
},
|
||||
"vertex_ai/claude-3-sonnet@20240229": {
|
||||
"max_tokens": 200000,
|
||||
"max_tokens": 4096,
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 4096,
|
||||
"input_cost_per_token": 0.000003,
|
||||
"output_cost_per_token": 0.000015,
|
||||
|
|
@ -1087,13 +1091,23 @@
|
|||
"mode": "chat"
|
||||
},
|
||||
"vertex_ai/claude-3-haiku@20240307": {
|
||||
"max_tokens": 200000,
|
||||
"max_tokens": 4096,
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 4096,
|
||||
"input_cost_per_token": 0.00000025,
|
||||
"output_cost_per_token": 0.00000125,
|
||||
"litellm_provider": "vertex_ai-anthropic_models",
|
||||
"mode": "chat"
|
||||
},
|
||||
"vertex_ai/claude-3-opus@20240229": {
|
||||
"max_tokens": 4096,
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 4096,
|
||||
"input_cost_per_token": 0.0000015,
|
||||
"output_cost_per_token": 0.0000075,
|
||||
"litellm_provider": "vertex_ai-anthropic_models",
|
||||
"mode": "chat"
|
||||
},
|
||||
"textembedding-gecko": {
|
||||
"max_tokens": 3072,
|
||||
"max_input_tokens": 3072,
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1 +0,0 @@
|
|||
!function(){"use strict";var e,t,n,r,o,u,i,c,f,a={},l={};function d(e){var t=l[e];if(void 0!==t)return t.exports;var n=l[e]={id:e,loaded:!1,exports:{}},r=!0;try{a[e](n,n.exports,d),r=!1}finally{r&&delete l[e]}return n.loaded=!0,n.exports}d.m=a,e=[],d.O=function(t,n,r,o){if(n){o=o||0;for(var u=e.length;u>0&&e[u-1][2]>o;u--)e[u]=e[u-1];e[u]=[n,r,o];return}for(var i=1/0,u=0;u<e.length;u++){for(var n=e[u][0],r=e[u][1],o=e[u][2],c=!0,f=0;f<n.length;f++)i>=o&&Object.keys(d.O).every(function(e){return d.O[e](n[f])})?n.splice(f--,1):(c=!1,o<i&&(i=o));if(c){e.splice(u--,1);var a=r();void 0!==a&&(t=a)}}return t},d.n=function(e){var t=e&&e.__esModule?function(){return e.default}:function(){return e};return d.d(t,{a:t}),t},n=Object.getPrototypeOf?function(e){return Object.getPrototypeOf(e)}:function(e){return e.__proto__},d.t=function(e,r){if(1&r&&(e=this(e)),8&r||"object"==typeof e&&e&&(4&r&&e.__esModule||16&r&&"function"==typeof e.then))return e;var o=Object.create(null);d.r(o);var u={};t=t||[null,n({}),n([]),n(n)];for(var i=2&r&&e;"object"==typeof i&&!~t.indexOf(i);i=n(i))Object.getOwnPropertyNames(i).forEach(function(t){u[t]=function(){return e[t]}});return u.default=function(){return e},d.d(o,u),o},d.d=function(e,t){for(var n in t)d.o(t,n)&&!d.o(e,n)&&Object.defineProperty(e,n,{enumerable:!0,get:t[n]})},d.f={},d.e=function(e){return Promise.all(Object.keys(d.f).reduce(function(t,n){return d.f[n](e,t),t},[]))},d.u=function(e){},d.miniCssF=function(e){return"static/css/a282d1bfd6ed4df8.css"},d.g=function(){if("object"==typeof globalThis)return globalThis;try{return this||Function("return this")()}catch(e){if("object"==typeof window)return window}}(),d.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)},r={},o="_N_E:",d.l=function(e,t,n,u){if(r[e]){r[e].push(t);return}if(void 0!==n)for(var i,c,f=document.getElementsByTagName("script"),a=0;a<f.length;a++){var l=f[a];if(l.getAttribute("src")==e||l.getAttribute("data-webpack")==o+n){i=l;break}}i||(c=!0,(i=document.createElement("script")).charset="utf-8",i.timeout=120,d.nc&&i.setAttribute("nonce",d.nc),i.setAttribute("data-webpack",o+n),i.src=d.tu(e)),r[e]=[t];var s=function(t,n){i.onerror=i.onload=null,clearTimeout(p);var o=r[e];if(delete r[e],i.parentNode&&i.parentNode.removeChild(i),o&&o.forEach(function(e){return e(n)}),t)return t(n)},p=setTimeout(s.bind(null,void 0,{type:"timeout",target:i}),12e4);i.onerror=s.bind(null,i.onerror),i.onload=s.bind(null,i.onload),c&&document.head.appendChild(i)},d.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},d.nmd=function(e){return e.paths=[],e.children||(e.children=[]),e},d.tt=function(){return void 0===u&&(u={createScriptURL:function(e){return e}},"undefined"!=typeof trustedTypes&&trustedTypes.createPolicy&&(u=trustedTypes.createPolicy("nextjs#bundler",u))),u},d.tu=function(e){return d.tt().createScriptURL(e)},d.p="/ui/_next/",i={272:0},d.f.j=function(e,t){var n=d.o(i,e)?i[e]:void 0;if(0!==n){if(n)t.push(n[2]);else if(272!=e){var r=new Promise(function(t,r){n=i[e]=[t,r]});t.push(n[2]=r);var o=d.p+d.u(e),u=Error();d.l(o,function(t){if(d.o(i,e)&&(0!==(n=i[e])&&(i[e]=void 0),n)){var r=t&&("load"===t.type?"missing":t.type),o=t&&t.target&&t.target.src;u.message="Loading chunk "+e+" failed.\n("+r+": "+o+")",u.name="ChunkLoadError",u.type=r,u.request=o,n[1](u)}},"chunk-"+e,e)}else i[e]=0}},d.O.j=function(e){return 0===i[e]},c=function(e,t){var n,r,o=t[0],u=t[1],c=t[2],f=0;if(o.some(function(e){return 0!==i[e]})){for(n in u)d.o(u,n)&&(d.m[n]=u[n]);if(c)var a=c(d)}for(e&&e(t);f<o.length;f++)r=o[f],d.o(i,r)&&i[r]&&i[r][0](),i[r]=0;return d.O(a)},(f=self.webpackChunk_N_E=self.webpackChunk_N_E||[]).forEach(c.bind(null,0)),f.push=c.bind(null,f.push.bind(f))}();
|
||||
|
|
@ -1 +1 @@
|
|||
!function(){"use strict";var e,t,n,r,o,u,i,c,f,a={},l={};function d(e){var t=l[e];if(void 0!==t)return t.exports;var n=l[e]={id:e,loaded:!1,exports:{}},r=!0;try{a[e](n,n.exports,d),r=!1}finally{r&&delete l[e]}return n.loaded=!0,n.exports}d.m=a,e=[],d.O=function(t,n,r,o){if(n){o=o||0;for(var u=e.length;u>0&&e[u-1][2]>o;u--)e[u]=e[u-1];e[u]=[n,r,o];return}for(var i=1/0,u=0;u<e.length;u++){for(var n=e[u][0],r=e[u][1],o=e[u][2],c=!0,f=0;f<n.length;f++)i>=o&&Object.keys(d.O).every(function(e){return d.O[e](n[f])})?n.splice(f--,1):(c=!1,o<i&&(i=o));if(c){e.splice(u--,1);var a=r();void 0!==a&&(t=a)}}return t},d.n=function(e){var t=e&&e.__esModule?function(){return e.default}:function(){return e};return d.d(t,{a:t}),t},n=Object.getPrototypeOf?function(e){return Object.getPrototypeOf(e)}:function(e){return e.__proto__},d.t=function(e,r){if(1&r&&(e=this(e)),8&r||"object"==typeof e&&e&&(4&r&&e.__esModule||16&r&&"function"==typeof e.then))return e;var o=Object.create(null);d.r(o);var u={};t=t||[null,n({}),n([]),n(n)];for(var i=2&r&&e;"object"==typeof i&&!~t.indexOf(i);i=n(i))Object.getOwnPropertyNames(i).forEach(function(t){u[t]=function(){return e[t]}});return u.default=function(){return e},d.d(o,u),o},d.d=function(e,t){for(var n in t)d.o(t,n)&&!d.o(e,n)&&Object.defineProperty(e,n,{enumerable:!0,get:t[n]})},d.f={},d.e=function(e){return Promise.all(Object.keys(d.f).reduce(function(t,n){return d.f[n](e,t),t},[]))},d.u=function(e){},d.miniCssF=function(e){return"static/css/a282d1bfd6ed4df8.css"},d.g=function(){if("object"==typeof globalThis)return globalThis;try{return this||Function("return this")()}catch(e){if("object"==typeof window)return window}}(),d.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)},r={},o="_N_E:",d.l=function(e,t,n,u){if(r[e]){r[e].push(t);return}if(void 0!==n)for(var i,c,f=document.getElementsByTagName("script"),a=0;a<f.length;a++){var l=f[a];if(l.getAttribute("src")==e||l.getAttribute("data-webpack")==o+n){i=l;break}}i||(c=!0,(i=document.createElement("script")).charset="utf-8",i.timeout=120,d.nc&&i.setAttribute("nonce",d.nc),i.setAttribute("data-webpack",o+n),i.src=d.tu(e)),r[e]=[t];var s=function(t,n){i.onerror=i.onload=null,clearTimeout(p);var o=r[e];if(delete r[e],i.parentNode&&i.parentNode.removeChild(i),o&&o.forEach(function(e){return e(n)}),t)return t(n)},p=setTimeout(s.bind(null,void 0,{type:"timeout",target:i}),12e4);i.onerror=s.bind(null,i.onerror),i.onload=s.bind(null,i.onload),c&&document.head.appendChild(i)},d.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},d.nmd=function(e){return e.paths=[],e.children||(e.children=[]),e},d.tt=function(){return void 0===u&&(u={createScriptURL:function(e){return e}},"undefined"!=typeof trustedTypes&&trustedTypes.createPolicy&&(u=trustedTypes.createPolicy("nextjs#bundler",u))),u},d.tu=function(e){return d.tt().createScriptURL(e)},d.p="/ui/_next/",i={272:0},d.f.j=function(e,t){var n=d.o(i,e)?i[e]:void 0;if(0!==n){if(n)t.push(n[2]);else if(272!=e){var r=new Promise(function(t,r){n=i[e]=[t,r]});t.push(n[2]=r);var o=d.p+d.u(e),u=Error();d.l(o,function(t){if(d.o(i,e)&&(0!==(n=i[e])&&(i[e]=void 0),n)){var r=t&&("load"===t.type?"missing":t.type),o=t&&t.target&&t.target.src;u.message="Loading chunk "+e+" failed.\n("+r+": "+o+")",u.name="ChunkLoadError",u.type=r,u.request=o,n[1](u)}},"chunk-"+e,e)}else i[e]=0}},d.O.j=function(e){return 0===i[e]},c=function(e,t){var n,r,o=t[0],u=t[1],c=t[2],f=0;if(o.some(function(e){return 0!==i[e]})){for(n in u)d.o(u,n)&&(d.m[n]=u[n]);if(c)var a=c(d)}for(e&&e(t);f<o.length;f++)r=o[f],d.o(i,r)&&i[r]&&i[r][0](),i[r]=0;return d.O(a)},(f=self.webpackChunk_N_E=self.webpackChunk_N_E||[]).forEach(c.bind(null,0)),f.push=c.bind(null,f.push.bind(f))}();
|
||||
!function(){"use strict";var e,t,n,r,o,u,i,c,f,a={},l={};function d(e){var t=l[e];if(void 0!==t)return t.exports;var n=l[e]={id:e,loaded:!1,exports:{}},r=!0;try{a[e](n,n.exports,d),r=!1}finally{r&&delete l[e]}return n.loaded=!0,n.exports}d.m=a,e=[],d.O=function(t,n,r,o){if(n){o=o||0;for(var u=e.length;u>0&&e[u-1][2]>o;u--)e[u]=e[u-1];e[u]=[n,r,o];return}for(var i=1/0,u=0;u<e.length;u++){for(var n=e[u][0],r=e[u][1],o=e[u][2],c=!0,f=0;f<n.length;f++)i>=o&&Object.keys(d.O).every(function(e){return d.O[e](n[f])})?n.splice(f--,1):(c=!1,o<i&&(i=o));if(c){e.splice(u--,1);var a=r();void 0!==a&&(t=a)}}return t},d.n=function(e){var t=e&&e.__esModule?function(){return e.default}:function(){return e};return d.d(t,{a:t}),t},n=Object.getPrototypeOf?function(e){return Object.getPrototypeOf(e)}:function(e){return e.__proto__},d.t=function(e,r){if(1&r&&(e=this(e)),8&r||"object"==typeof e&&e&&(4&r&&e.__esModule||16&r&&"function"==typeof e.then))return e;var o=Object.create(null);d.r(o);var u={};t=t||[null,n({}),n([]),n(n)];for(var i=2&r&&e;"object"==typeof i&&!~t.indexOf(i);i=n(i))Object.getOwnPropertyNames(i).forEach(function(t){u[t]=function(){return e[t]}});return u.default=function(){return e},d.d(o,u),o},d.d=function(e,t){for(var n in t)d.o(t,n)&&!d.o(e,n)&&Object.defineProperty(e,n,{enumerable:!0,get:t[n]})},d.f={},d.e=function(e){return Promise.all(Object.keys(d.f).reduce(function(t,n){return d.f[n](e,t),t},[]))},d.u=function(e){},d.miniCssF=function(e){return"static/css/11cfce8bfdf6e8f1.css"},d.g=function(){if("object"==typeof globalThis)return globalThis;try{return this||Function("return this")()}catch(e){if("object"==typeof window)return window}}(),d.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)},r={},o="_N_E:",d.l=function(e,t,n,u){if(r[e]){r[e].push(t);return}if(void 0!==n)for(var i,c,f=document.getElementsByTagName("script"),a=0;a<f.length;a++){var l=f[a];if(l.getAttribute("src")==e||l.getAttribute("data-webpack")==o+n){i=l;break}}i||(c=!0,(i=document.createElement("script")).charset="utf-8",i.timeout=120,d.nc&&i.setAttribute("nonce",d.nc),i.setAttribute("data-webpack",o+n),i.src=d.tu(e)),r[e]=[t];var s=function(t,n){i.onerror=i.onload=null,clearTimeout(p);var o=r[e];if(delete r[e],i.parentNode&&i.parentNode.removeChild(i),o&&o.forEach(function(e){return e(n)}),t)return t(n)},p=setTimeout(s.bind(null,void 0,{type:"timeout",target:i}),12e4);i.onerror=s.bind(null,i.onerror),i.onload=s.bind(null,i.onload),c&&document.head.appendChild(i)},d.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},d.nmd=function(e){return e.paths=[],e.children||(e.children=[]),e},d.tt=function(){return void 0===u&&(u={createScriptURL:function(e){return e}},"undefined"!=typeof trustedTypes&&trustedTypes.createPolicy&&(u=trustedTypes.createPolicy("nextjs#bundler",u))),u},d.tu=function(e){return d.tt().createScriptURL(e)},d.p="/ui/_next/",i={272:0},d.f.j=function(e,t){var n=d.o(i,e)?i[e]:void 0;if(0!==n){if(n)t.push(n[2]);else if(272!=e){var r=new Promise(function(t,r){n=i[e]=[t,r]});t.push(n[2]=r);var o=d.p+d.u(e),u=Error();d.l(o,function(t){if(d.o(i,e)&&(0!==(n=i[e])&&(i[e]=void 0),n)){var r=t&&("load"===t.type?"missing":t.type),o=t&&t.target&&t.target.src;u.message="Loading chunk "+e+" failed.\n("+r+": "+o+")",u.name="ChunkLoadError",u.type=r,u.request=o,n[1](u)}},"chunk-"+e,e)}else i[e]=0}},d.O.j=function(e){return 0===i[e]},c=function(e,t){var n,r,o=t[0],u=t[1],c=t[2],f=0;if(o.some(function(e){return 0!==i[e]})){for(n in u)d.o(u,n)&&(d.m[n]=u[n]);if(c)var a=c(d)}for(e&&e(t);f<o.length;f++)r=o[f],d.o(i,r)&&i[r]&&i[r][0](),i[r]=0;return d.O(a)},(f=self.webpackChunk_N_E=self.webpackChunk_N_E||[]).forEach(c.bind(null,0)),f.push=c.bind(null,f.push.bind(f))}();
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1 +1 @@
|
|||
<!DOCTYPE html><html id="__next_error__"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/ui/_next/static/chunks/webpack-11b043d6a7ef78fa.js" crossorigin=""/><script src="/ui/_next/static/chunks/fd9d1056-a507ee9e75a3be72.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/69-589b47e7a69d316f.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/main-app-096338c8e1915716.js" async="" crossorigin=""></script><title>LiteLLM Dashboard</title><meta name="description" content="LiteLLM Proxy Admin UI"/><link rel="icon" href="/ui/favicon.ico" type="image/x-icon" sizes="16x16"/><meta name="next-size-adjust"/><script src="/ui/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js" crossorigin="" noModule=""></script></head><body><script src="/ui/_next/static/chunks/webpack-11b043d6a7ef78fa.js" crossorigin="" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/ui/_next/static/media/c9a5bc6a7c948fb0-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n2:HL[\"/ui/_next/static/css/a282d1bfd6ed4df8.css\",\"style\",{\"crossOrigin\":\"\"}]\n0:\"$L3\"\n"])</script><script>self.__next_f.push([1,"4:I[47690,[],\"\"]\n6:I[77831,[],\"\"]\n7:I[29306,[\"823\",\"static/chunks/823-2ada48e2e6a5ab39.js\",\"931\",\"static/chunks/app/page-e16bcf8bdc356530.js\"],\"\"]\n8:I[5613,[],\"\"]\n9:I[31778,[],\"\"]\nb:I[48955,[],\"\"]\nc:[]\n"])</script><script>self.__next_f.push([1,"3:[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/ui/_next/static/css/a282d1bfd6ed4df8.css\",\"precedence\":\"next\",\"crossOrigin\":\"\"}]],[\"$\",\"$L4\",null,{\"buildId\":\"BNBzATtnAelV8BpmzRdfL\",\"assetPrefix\":\"/ui\",\"initialCanonicalUrl\":\"/\",\"initialTree\":[\"\",{\"children\":[\"__PAGE__\",{}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"__PAGE__\",{},[\"$L5\",[\"$\",\"$L6\",null,{\"propsForComponent\":{\"params\":{}},\"Component\":\"$7\",\"isStaticGeneration\":true}],null]]},[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"className\":\"__className_c23dc8\",\"children\":[\"$\",\"$L8\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"loadingScripts\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L9\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[],\"styles\":null}]}]}],null]],\"initialHead\":[false,\"$La\"],\"globalErrorComponent\":\"$b\",\"missingSlots\":\"$Wc\"}]]\n"])</script><script>self.__next_f.push([1,"a:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"LiteLLM Dashboard\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"LiteLLM Proxy Admin UI\"}],[\"$\",\"link\",\"4\",{\"rel\":\"icon\",\"href\":\"/ui/favicon.ico\",\"type\":\"image/x-icon\",\"sizes\":\"16x16\"}],[\"$\",\"meta\",\"5\",{\"name\":\"next-size-adjust\"}]]\n5:null\n"])</script><script>self.__next_f.push([1,""])</script></body></html>
|
||||
<!DOCTYPE html><html id="__next_error__"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/ui/_next/static/chunks/webpack-59f93936973f5f5a.js" crossorigin=""/><script src="/ui/_next/static/chunks/fd9d1056-bcf69420342937de.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/69-442a9c01c3fd20f9.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/main-app-096338c8e1915716.js" async="" crossorigin=""></script><title>LiteLLM Dashboard</title><meta name="description" content="LiteLLM Proxy Admin UI"/><link rel="icon" href="/ui/favicon.ico" type="image/x-icon" sizes="16x16"/><meta name="next-size-adjust"/><script src="/ui/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js" crossorigin="" noModule=""></script></head><body><script src="/ui/_next/static/chunks/webpack-59f93936973f5f5a.js" crossorigin="" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/ui/_next/static/media/c9a5bc6a7c948fb0-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n2:HL[\"/ui/_next/static/css/11cfce8bfdf6e8f1.css\",\"style\",{\"crossOrigin\":\"\"}]\n0:\"$L3\"\n"])</script><script>self.__next_f.push([1,"4:I[47690,[],\"\"]\n6:I[77831,[],\"\"]\n7:I[44488,[\"289\",\"static/chunks/289-04be6cb9636840d2.js\",\"931\",\"static/chunks/app/page-3dcbe3938de7e459.js\"],\"\"]\n8:I[5613,[],\"\"]\n9:I[31778,[],\"\"]\nb:I[48955,[],\"\"]\nc:[]\n"])</script><script>self.__next_f.push([1,"3:[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/ui/_next/static/css/11cfce8bfdf6e8f1.css\",\"precedence\":\"next\",\"crossOrigin\":\"\"}]],[\"$\",\"$L4\",null,{\"buildId\":\"G0uskIep5NGJbFqI3_DY1\",\"assetPrefix\":\"/ui\",\"initialCanonicalUrl\":\"/\",\"initialTree\":[\"\",{\"children\":[\"__PAGE__\",{}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"__PAGE__\",{},[\"$L5\",[\"$\",\"$L6\",null,{\"propsForComponent\":{\"params\":{}},\"Component\":\"$7\",\"isStaticGeneration\":true}],null]]},[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"className\":\"__className_c23dc8\",\"children\":[\"$\",\"$L8\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"loadingScripts\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L9\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[],\"styles\":null}]}]}],null]],\"initialHead\":[false,\"$La\"],\"globalErrorComponent\":\"$b\",\"missingSlots\":\"$Wc\"}]]\n"])</script><script>self.__next_f.push([1,"a:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"LiteLLM Dashboard\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"LiteLLM Proxy Admin UI\"}],[\"$\",\"link\",\"4\",{\"rel\":\"icon\",\"href\":\"/ui/favicon.ico\",\"type\":\"image/x-icon\",\"sizes\":\"16x16\"}],[\"$\",\"meta\",\"5\",{\"name\":\"next-size-adjust\"}]]\n5:null\n"])</script><script>self.__next_f.push([1,""])</script></body></html>
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
2:I[77831,[],""]
|
||||
3:I[29306,["823","static/chunks/823-2ada48e2e6a5ab39.js","931","static/chunks/app/page-e16bcf8bdc356530.js"],""]
|
||||
3:I[44488,["289","static/chunks/289-04be6cb9636840d2.js","931","static/chunks/app/page-3dcbe3938de7e459.js"],""]
|
||||
4:I[5613,[],""]
|
||||
5:I[31778,[],""]
|
||||
0:["BNBzATtnAelV8BpmzRdfL",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},["$L1",["$","$L2",null,{"propsForComponent":{"params":{}},"Component":"$3","isStaticGeneration":true}],null]]},[null,["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_c23dc8","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"loading":"$undefined","loadingStyles":"$undefined","loadingScripts":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[],"styles":null}]}]}],null]],[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/a282d1bfd6ed4df8.css","precedence":"next","crossOrigin":""}]],"$L6"]]]]
|
||||
0:["G0uskIep5NGJbFqI3_DY1",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},["$L1",["$","$L2",null,{"propsForComponent":{"params":{}},"Component":"$3","isStaticGeneration":true}],null]]},[null,["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_c23dc8","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"loading":"$undefined","loadingStyles":"$undefined","loadingScripts":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[],"styles":null}]}]}],null]],[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/11cfce8bfdf6e8f1.css","precedence":"next","crossOrigin":""}]],"$L6"]]]]
|
||||
6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/ui/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","meta","5",{"name":"next-size-adjust"}]]
|
||||
1:null
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ model_list:
|
|||
|
||||
litellm_settings:
|
||||
success_callback: ["prometheus"]
|
||||
service_callback: ["prometheus_system"]
|
||||
upperbound_key_generate_params:
|
||||
max_budget: os.environ/LITELLM_UPPERBOUND_KEYS_MAX_BUDGET
|
||||
|
||||
|
|
@ -39,7 +40,7 @@ general_settings:
|
|||
master_key: sk-1234
|
||||
allow_user_auth: true
|
||||
alerting: ["slack"]
|
||||
# store_model_in_db: True // set via environment variable - os.environ["STORE_MODEL_IN_DB"] = "True"
|
||||
store_model_in_db: True // set via environment variable - os.environ["STORE_MODEL_IN_DB"] = "True"
|
||||
proxy_batch_write_at: 5 # 👈 Frequency of batch writing logs to server (in seconds)
|
||||
enable_jwt_auth: True
|
||||
alerting: ["slack"]
|
||||
|
|
|
|||
|
|
@ -458,8 +458,16 @@ class TeamMemberDeleteRequest(LiteLLMBase):
|
|||
return values
|
||||
|
||||
|
||||
class UpdateTeamRequest(TeamBase):
|
||||
class UpdateTeamRequest(LiteLLMBase):
|
||||
team_id: str # required
|
||||
team_alias: Optional[str] = None
|
||||
organization_id: Optional[str] = None
|
||||
metadata: Optional[dict] = None
|
||||
tpm_limit: Optional[int] = None
|
||||
rpm_limit: Optional[int] = None
|
||||
max_budget: Optional[float] = None
|
||||
models: Optional[list] = None
|
||||
blocked: Optional[bool] = None
|
||||
|
||||
|
||||
class DeleteTeamRequest(LiteLLMBase):
|
||||
|
|
|
|||
|
|
@ -86,7 +86,12 @@ async def perform_health_check(
|
|||
return [], []
|
||||
|
||||
if model is not None:
|
||||
model_list = [x for x in model_list if x["litellm_params"]["model"] == model]
|
||||
_new_model_list = [
|
||||
x for x in model_list if x["litellm_params"]["model"] == model
|
||||
]
|
||||
if _new_model_list == []:
|
||||
_new_model_list = [x for x in model_list if x["model_name"] == model]
|
||||
model_list = _new_model_list
|
||||
|
||||
healthy_endpoints, unhealthy_endpoints = await _perform_health_check(model_list)
|
||||
|
||||
|
|
|
|||
|
|
@ -4,6 +4,23 @@ model_list:
|
|||
model: openai/fake
|
||||
api_key: fake-key
|
||||
api_base: https://exampleopenaiendpoint-production.up.railway.app/
|
||||
- model_name: "*"
|
||||
litellm_params:
|
||||
model: openai/*
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
|
||||
|
||||
litellm_settings:
|
||||
default_team_settings:
|
||||
- team_id: team-1
|
||||
success_callback: ["langfuse"]
|
||||
langfuse_public_key: os.environ/LANGFUSE_PROJECT1_PUBLIC # Project 1
|
||||
langfuse_secret: os.environ/LANGFUSE_PROJECT1_SECRET # Project 1
|
||||
- team_id: team-2
|
||||
success_callback: ["langfuse"]
|
||||
langfuse_public_key: os.environ/LANGFUSE_PROJECT2_PUBLIC # Project 2
|
||||
langfuse_secret: os.environ/LANGFUSE_PROJECT2_SECRET # Project 2
|
||||
|
||||
general_settings:
|
||||
store_model_in_db: true
|
||||
master_key: sk-1234
|
||||
|
|
@ -703,7 +703,11 @@ async def user_api_key_auth(
|
|||
verbose_proxy_logger.debug(
|
||||
f"model: {model}; allowed_models: {filtered_models}"
|
||||
)
|
||||
if model is not None and model not in filtered_models:
|
||||
if (
|
||||
model is not None
|
||||
and model not in filtered_models
|
||||
and "*" not in filtered_models
|
||||
):
|
||||
raise ValueError(
|
||||
f"API Key not allowed to access model. This token can only access models={valid_token.models}. Tried to access {model}"
|
||||
)
|
||||
|
|
@ -1896,7 +1900,12 @@ class ProxyConfig:
|
|||
param_name = getattr(response, "param_name", None)
|
||||
param_value = getattr(response, "param_value", None)
|
||||
if param_name is not None and param_value is not None:
|
||||
config[param_name] = param_value
|
||||
# check if param_name is already in the config
|
||||
if param_name in config:
|
||||
if isinstance(config[param_name], dict):
|
||||
config[param_name].update(param_value)
|
||||
else:
|
||||
config[param_name] = param_value
|
||||
|
||||
return config
|
||||
|
||||
|
|
@ -2441,7 +2450,7 @@ class ProxyConfig:
|
|||
- Check if model id's in router already
|
||||
- If not, add to router
|
||||
"""
|
||||
global llm_router, llm_model_list, master_key
|
||||
global llm_router, llm_model_list, master_key, general_settings
|
||||
|
||||
import base64
|
||||
|
||||
|
|
@ -2494,9 +2503,6 @@ class ProxyConfig:
|
|||
verbose_proxy_logger.debug(f"_model_list: {_model_list}")
|
||||
llm_router = litellm.Router(model_list=_model_list)
|
||||
verbose_proxy_logger.debug(f"updated llm_router: {llm_router}")
|
||||
verbose_proxy_logger.debug(
|
||||
f"llm_router model list: {llm_router.model_list}"
|
||||
)
|
||||
else:
|
||||
new_models = await prisma_client.db.litellm_proxymodeltable.find_many()
|
||||
verbose_proxy_logger.debug(f"len new_models: {len(new_models)}")
|
||||
|
|
@ -2541,7 +2547,7 @@ class ProxyConfig:
|
|||
config_data = await proxy_config.get_config()
|
||||
litellm_settings = config_data.get("litellm_settings", {}) or {}
|
||||
success_callbacks = litellm_settings.get("success_callback", None)
|
||||
_added_callback = False
|
||||
|
||||
if success_callbacks is not None and isinstance(success_callbacks, list):
|
||||
for success_callback in success_callbacks:
|
||||
if success_callback not in litellm.success_callback:
|
||||
|
|
@ -2557,6 +2563,13 @@ class ProxyConfig:
|
|||
verbose_proxy_logger.error(
|
||||
"Error setting env variable: %s - %s", k, str(e)
|
||||
)
|
||||
|
||||
# general_settings
|
||||
_general_settings = config_data.get("general_settings", {})
|
||||
if "alerting" in _general_settings:
|
||||
general_settings["alerting"] = _general_settings["alerting"]
|
||||
proxy_logging_obj.alerting = general_settings["alerting"]
|
||||
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.error(
|
||||
"{}\nTraceback:{}".format(str(e), traceback.format_exc())
|
||||
|
|
@ -3186,6 +3199,11 @@ async def startup_event():
|
|||
seconds=30,
|
||||
args=[prisma_client, proxy_logging_obj],
|
||||
)
|
||||
|
||||
# this will load all existing models on proxy startup
|
||||
await proxy_config.add_deployment(
|
||||
prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj
|
||||
)
|
||||
scheduler.start()
|
||||
|
||||
|
||||
|
|
@ -6301,7 +6319,7 @@ async def update_team(
|
|||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
):
|
||||
"""
|
||||
[RECOMMENDED] - use `/team/member_add` to add new team members instead
|
||||
Use `/team/member_add` AND `/team/member/delete` to add/remove new team members
|
||||
|
||||
You can now update team budget / rate limits via /team/update
|
||||
|
||||
|
|
@ -6351,60 +6369,6 @@ async def update_team(
|
|||
team_id=data.team_id,
|
||||
)
|
||||
|
||||
## ADD NEW USERS ##
|
||||
existing_user_id_list = []
|
||||
## Get new users
|
||||
for user in existing_team_row.members_with_roles:
|
||||
if user["user_id"] is not None:
|
||||
existing_user_id_list.append(user["user_id"])
|
||||
|
||||
## Update new user rows with team id (info used by /user/info to show all teams, user is a part of)
|
||||
if data.members_with_roles is not None:
|
||||
for user in data.members_with_roles:
|
||||
if user.user_id not in existing_user_id_list:
|
||||
await prisma_client.update_data(
|
||||
user_id=user.user_id,
|
||||
data={
|
||||
"user_id": user.user_id,
|
||||
"teams": [team_row["team_id"]],
|
||||
"models": team_row["data"].models,
|
||||
},
|
||||
update_key_values_custom_query={
|
||||
"teams": {
|
||||
"push": [team_row["team_id"]],
|
||||
}
|
||||
},
|
||||
table_name="user",
|
||||
)
|
||||
|
||||
## REMOVE DELETED USERS ##
|
||||
### Get list of deleted users (old list - new list)
|
||||
deleted_user_id_list = []
|
||||
new_user_id_list = []
|
||||
## Get old user list
|
||||
if data.members_with_roles is not None:
|
||||
for user in data.members_with_roles:
|
||||
new_user_id_list.append(user.user_id)
|
||||
## Get diff
|
||||
if existing_team_row.members_with_roles is not None:
|
||||
for user in existing_team_row.members_with_roles:
|
||||
if user["user_id"] not in new_user_id_list and len(new_user_id_list) > 0:
|
||||
deleted_user_id_list.append(user["user_id"])
|
||||
|
||||
## SET UPDATED LIST
|
||||
if len(deleted_user_id_list) > 0:
|
||||
# get the deleted users
|
||||
existing_user_rows = await prisma_client.get_data(
|
||||
user_id_list=deleted_user_id_list, table_name="user", query_type="find_all"
|
||||
)
|
||||
for user in existing_user_rows:
|
||||
if data.team_id in user["teams"]:
|
||||
user["teams"].remove(data.team_id)
|
||||
await prisma_client.update_data(
|
||||
user_id=user["user_id"],
|
||||
data=user,
|
||||
update_key_values={"user_id": user["user_id"], "teams": user["teams"]},
|
||||
)
|
||||
return team_row
|
||||
|
||||
|
||||
|
|
@ -7152,8 +7116,9 @@ async def model_info_v2(
|
|||
if k not in model_info:
|
||||
model_info[k] = v
|
||||
model["model_info"] = model_info
|
||||
# don't return the api key
|
||||
# don't return the api key / vertex credentials
|
||||
model["litellm_params"].pop("api_key", None)
|
||||
model["litellm_params"].pop("vertex_credentials", None)
|
||||
|
||||
verbose_proxy_logger.debug("all_models: %s", all_models)
|
||||
return {"data": all_models}
|
||||
|
|
@ -8118,7 +8083,7 @@ async def update_config(config_info: ConfigYAML):
|
|||
|
||||
Currently supports modifying General Settings + LiteLLM settings
|
||||
"""
|
||||
global llm_router, llm_model_list, general_settings, proxy_config, proxy_logging_obj, master_key
|
||||
global llm_router, llm_model_list, general_settings, proxy_config, proxy_logging_obj, master_key, prisma_client
|
||||
try:
|
||||
import base64
|
||||
|
||||
|
|
@ -8186,6 +8151,12 @@ async def update_config(config_info: ConfigYAML):
|
|||
# Save the updated config
|
||||
await proxy_config.save_config(new_config=config)
|
||||
|
||||
# make sure the change is instantly rolled out for langfuse
|
||||
if prisma_client is not None:
|
||||
await proxy_config.add_deployment(
|
||||
prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj
|
||||
)
|
||||
|
||||
# Test new connections
|
||||
## Slack
|
||||
if "slack" in config.get("general_settings", {}).get("alerting", []):
|
||||
|
|
@ -8221,15 +8192,80 @@ async def update_config(config_info: ConfigYAML):
|
|||
async def get_config():
|
||||
"""
|
||||
For Admin UI - allows admin to view config via UI
|
||||
# return the callbacks and the env variables for the callback
|
||||
|
||||
"""
|
||||
global llm_router, llm_model_list, general_settings, proxy_config, proxy_logging_obj, master_key
|
||||
try:
|
||||
import base64
|
||||
|
||||
config_data = await proxy_config.get_config()
|
||||
config_data = config_data.get("litellm_settings", {})
|
||||
_litellm_settings = config_data.get("litellm_settings", {})
|
||||
_general_settings = config_data.get("general_settings", {})
|
||||
environment_variables = config_data.get("environment_variables", {})
|
||||
|
||||
return {"data": config_data, "status": "success"}
|
||||
# check if "langfuse" in litellm_settings
|
||||
_success_callbacks = _litellm_settings.get("success_callback", [])
|
||||
_data_to_return = []
|
||||
"""
|
||||
[
|
||||
{
|
||||
"name": "langfuse",
|
||||
"variables": {
|
||||
"LANGFUSE_PUB_KEY": "value",
|
||||
"LANGFUSE_SECRET_KEY": "value",
|
||||
"LANGFUSE_HOST": "value"
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
"""
|
||||
for _callback in _success_callbacks:
|
||||
if _callback == "langfuse":
|
||||
_langfuse_vars = [
|
||||
"LANGFUSE_PUBLIC_KEY",
|
||||
"LANGFUSE_SECRET_KEY",
|
||||
"LANGFUSE_HOST",
|
||||
]
|
||||
_langfuse_env_vars = {}
|
||||
for _var in _langfuse_vars:
|
||||
env_variable = environment_variables.get(_var, None)
|
||||
if env_variable is None:
|
||||
_langfuse_env_vars[_var] = None
|
||||
else:
|
||||
# decode + decrypt the value
|
||||
decoded_b64 = base64.b64decode(env_variable)
|
||||
_decrypted_value = decrypt_value(
|
||||
value=decoded_b64, master_key=master_key
|
||||
)
|
||||
_langfuse_env_vars[_var] = _decrypted_value
|
||||
|
||||
_data_to_return.append(
|
||||
{"name": _callback, "variables": _langfuse_env_vars}
|
||||
)
|
||||
|
||||
# Check if slack alerting is on
|
||||
_alerting = _general_settings.get("alerting", [])
|
||||
if "slack" in _alerting:
|
||||
_slack_vars = [
|
||||
"SLACK_WEBHOOK_URL",
|
||||
]
|
||||
_slack_env_vars = {}
|
||||
for _var in _slack_vars:
|
||||
env_variable = environment_variables.get(_var, None)
|
||||
if env_variable is None:
|
||||
_slack_env_vars[_var] = None
|
||||
else:
|
||||
# decode + decrypt the value
|
||||
decoded_b64 = base64.b64decode(env_variable)
|
||||
_decrypted_value = decrypt_value(
|
||||
value=decoded_b64, master_key=master_key
|
||||
)
|
||||
_slack_env_vars[_var] = _decrypted_value
|
||||
|
||||
_data_to_return.append({"name": "slack", "variables": _slack_env_vars})
|
||||
|
||||
return {"status": "success", "data": _data_to_return}
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
if isinstance(e, HTTPException):
|
||||
|
|
@ -8303,7 +8339,7 @@ async def test_endpoint(request: Request):
|
|||
)
|
||||
async def health_services_endpoint(
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
service: Literal["slack_budget_alerts"] = fastapi.Query(
|
||||
service: Literal["slack_budget_alerts", "langfuse", "slack"] = fastapi.Query(
|
||||
description="Specify the service being hit."
|
||||
),
|
||||
):
|
||||
|
|
@ -8319,8 +8355,7 @@ async def health_services_endpoint(
|
|||
raise HTTPException(
|
||||
status_code=400, detail={"error": "Service must be specified."}
|
||||
)
|
||||
|
||||
if service not in ["slack_budget_alerts"]:
|
||||
if service not in ["slack_budget_alerts", "langfuse", "slack"]:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
|
|
@ -8328,9 +8363,29 @@ async def health_services_endpoint(
|
|||
},
|
||||
)
|
||||
|
||||
if service == "langfuse":
|
||||
from litellm.integrations.langfuse import LangFuseLogger
|
||||
|
||||
langfuse_logger = LangFuseLogger()
|
||||
langfuse_logger.Langfuse.auth_check()
|
||||
_ = litellm.completion(
|
||||
model="openai/litellm-mock-response-model",
|
||||
messages=[{"role": "user", "content": "Hey, how's it going?"}],
|
||||
user="litellm:/health/services",
|
||||
mock_response="This is a mock response",
|
||||
)
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "Mock LLM request made - check langfuse.",
|
||||
}
|
||||
|
||||
if "slack" in general_settings.get("alerting", []):
|
||||
test_message = f"""\n🚨 `ProjectedLimitExceededError` 💸\n\n`Key Alias:` litellm-ui-test-alert \n`Expected Day of Error`: 28th March \n`Current Spend`: $100.00 \n`Projected Spend at end of month`: $1000.00 \n`Soft Limit`: $700"""
|
||||
await proxy_logging_obj.alerting_handler(message=test_message, level="Low")
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "Mock Slack Alert sent, verify Slack Alert Received on your channel",
|
||||
}
|
||||
else:
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
|
|
|
|||
|
|
@ -1103,6 +1103,7 @@ class Router:
|
|||
raise e
|
||||
|
||||
async def _aembedding(self, input: Union[str, List], model: str, **kwargs):
|
||||
model_name = None
|
||||
try:
|
||||
verbose_router_logger.debug(
|
||||
f"Inside _aembedding()- model: {model}; kwargs: {kwargs}"
|
||||
|
|
@ -2306,7 +2307,9 @@ class Router:
|
|||
return self.model_names
|
||||
|
||||
def get_model_list(self):
|
||||
return self.model_list
|
||||
if hasattr(self, "model_list"):
|
||||
return self.model_list
|
||||
return None
|
||||
|
||||
def _get_client(self, deployment, kwargs, client_type=None):
|
||||
"""
|
||||
|
|
@ -2774,7 +2777,10 @@ class Router:
|
|||
}
|
||||
else:
|
||||
# check response_ms and update num_successes
|
||||
response_ms = response.get("_response_ms", 0)
|
||||
if isinstance(response, dict):
|
||||
response_ms = response.get("_response_ms", 0)
|
||||
else:
|
||||
response_ms = 0
|
||||
if model_id is not None:
|
||||
if model_id in self.deployment_stats:
|
||||
# check if avg_latency exists
|
||||
|
|
|
|||
|
|
@ -23,6 +23,40 @@ user_message = "Write a short poem about the sky"
|
|||
messages = [{"content": user_message, "role": "user"}]
|
||||
|
||||
|
||||
def get_vertex_ai_creds_json() -> dict:
|
||||
# Define the path to the vertex_key.json file
|
||||
print("loading vertex ai credentials")
|
||||
filepath = os.path.dirname(os.path.abspath(__file__))
|
||||
vertex_key_path = filepath + "/vertex_key.json"
|
||||
|
||||
# Read the existing content of the file or create an empty dictionary
|
||||
try:
|
||||
with open(vertex_key_path, "r") as file:
|
||||
# Read the file content
|
||||
print("Read vertexai file path")
|
||||
content = file.read()
|
||||
|
||||
# If the file is empty or not valid JSON, create an empty dictionary
|
||||
if not content or not content.strip():
|
||||
service_account_key_data = {}
|
||||
else:
|
||||
# Attempt to load the existing JSON content
|
||||
file.seek(0)
|
||||
service_account_key_data = json.load(file)
|
||||
except FileNotFoundError:
|
||||
# If the file doesn't exist, create an empty dictionary
|
||||
service_account_key_data = {}
|
||||
|
||||
# Update the service_account_key_data with environment variables
|
||||
private_key_id = os.environ.get("VERTEX_AI_PRIVATE_KEY_ID", "")
|
||||
private_key = os.environ.get("VERTEX_AI_PRIVATE_KEY", "")
|
||||
private_key = private_key.replace("\\n", "\n")
|
||||
service_account_key_data["private_key_id"] = private_key_id
|
||||
service_account_key_data["private_key"] = private_key
|
||||
|
||||
return service_account_key_data
|
||||
|
||||
|
||||
def load_vertex_ai_credentials():
|
||||
# Define the path to the vertex_key.json file
|
||||
print("loading vertex ai credentials")
|
||||
|
|
@ -85,16 +119,16 @@ async def get_response():
|
|||
pytest.fail(f"An error occurred - {str(e)}")
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="Local test. Vertex AI Quota is low. Leads to rate limit errors on ci/cd."
|
||||
)
|
||||
# @pytest.mark.skip(
|
||||
# reason="Local test. Vertex AI Quota is low. Leads to rate limit errors on ci/cd."
|
||||
# )
|
||||
def test_vertex_ai_anthropic():
|
||||
load_vertex_ai_credentials()
|
||||
|
||||
model = "claude-3-sonnet@20240229"
|
||||
|
||||
vertex_ai_project = "adroit-crow-413218"
|
||||
vertex_ai_location = "asia-southeast1"
|
||||
json_obj = get_vertex_ai_creds_json()
|
||||
vertex_credentials = json.dumps(json_obj)
|
||||
|
||||
response = completion(
|
||||
model="vertex_ai/" + model,
|
||||
|
|
@ -102,15 +136,16 @@ def test_vertex_ai_anthropic():
|
|||
temperature=0.7,
|
||||
vertex_ai_project=vertex_ai_project,
|
||||
vertex_ai_location=vertex_ai_location,
|
||||
vertex_credentials=vertex_credentials,
|
||||
)
|
||||
print("\nModel Response", response)
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="Local test. Vertex AI Quota is low. Leads to rate limit errors on ci/cd."
|
||||
)
|
||||
# @pytest.mark.skip(
|
||||
# reason="Local test. Vertex AI Quota is low. Leads to rate limit errors on ci/cd."
|
||||
# )
|
||||
def test_vertex_ai_anthropic_streaming():
|
||||
load_vertex_ai_credentials()
|
||||
# load_vertex_ai_credentials()
|
||||
|
||||
# litellm.set_verbose = True
|
||||
|
||||
|
|
@ -118,6 +153,8 @@ def test_vertex_ai_anthropic_streaming():
|
|||
|
||||
vertex_ai_project = "adroit-crow-413218"
|
||||
vertex_ai_location = "asia-southeast1"
|
||||
json_obj = get_vertex_ai_creds_json()
|
||||
vertex_credentials = json.dumps(json_obj)
|
||||
|
||||
response = completion(
|
||||
model="vertex_ai/" + model,
|
||||
|
|
@ -137,17 +174,19 @@ def test_vertex_ai_anthropic_streaming():
|
|||
# test_vertex_ai_anthropic_streaming()
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="Local test. Vertex AI Quota is low. Leads to rate limit errors on ci/cd."
|
||||
)
|
||||
# @pytest.mark.skip(
|
||||
# reason="Local test. Vertex AI Quota is low. Leads to rate limit errors on ci/cd."
|
||||
# )
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_ai_anthropic_async():
|
||||
load_vertex_ai_credentials()
|
||||
# load_vertex_ai_credentials()
|
||||
|
||||
model = "claude-3-sonnet@20240229"
|
||||
|
||||
vertex_ai_project = "adroit-crow-413218"
|
||||
vertex_ai_location = "asia-southeast1"
|
||||
json_obj = get_vertex_ai_creds_json()
|
||||
vertex_credentials = json.dumps(json_obj)
|
||||
|
||||
response = await acompletion(
|
||||
model="vertex_ai/" + model,
|
||||
|
|
@ -155,6 +194,7 @@ async def test_vertex_ai_anthropic_async():
|
|||
temperature=0.7,
|
||||
vertex_ai_project=vertex_ai_project,
|
||||
vertex_ai_location=vertex_ai_location,
|
||||
vertex_credentials=vertex_credentials,
|
||||
)
|
||||
print(f"Model Response: {response}")
|
||||
|
||||
|
|
@ -162,17 +202,19 @@ async def test_vertex_ai_anthropic_async():
|
|||
# asyncio.run(test_vertex_ai_anthropic_async())
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="Local test. Vertex AI Quota is low. Leads to rate limit errors on ci/cd."
|
||||
)
|
||||
# @pytest.mark.skip(
|
||||
# reason="Local test. Vertex AI Quota is low. Leads to rate limit errors on ci/cd."
|
||||
# )
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_ai_anthropic_async_streaming():
|
||||
load_vertex_ai_credentials()
|
||||
|
||||
# load_vertex_ai_credentials()
|
||||
litellm.set_verbose = True
|
||||
model = "claude-3-sonnet@20240229"
|
||||
|
||||
vertex_ai_project = "adroit-crow-413218"
|
||||
vertex_ai_location = "asia-southeast1"
|
||||
json_obj = get_vertex_ai_creds_json()
|
||||
vertex_credentials = json.dumps(json_obj)
|
||||
|
||||
response = await acompletion(
|
||||
model="vertex_ai/" + model,
|
||||
|
|
@ -180,6 +222,7 @@ async def test_vertex_ai_anthropic_async_streaming():
|
|||
temperature=0.7,
|
||||
vertex_ai_project=vertex_ai_project,
|
||||
vertex_ai_location=vertex_ai_location,
|
||||
vertex_credentials=vertex_credentials,
|
||||
stream=True,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -492,6 +492,31 @@ def test_completion_claude2_1():
|
|||
|
||||
# test_completion_claude2_1()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_acompletion_claude2_1():
|
||||
try:
|
||||
litellm.set_verbose = True
|
||||
print("claude2.1 test request")
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "Your goal is generate a joke on the topic user gives.",
|
||||
},
|
||||
{"role": "user", "content": "Generate a 3 liner joke for me"},
|
||||
]
|
||||
# test without max tokens
|
||||
response = await litellm.acompletion(model="claude-2.1", messages=messages)
|
||||
# Add any assertions here to check the response
|
||||
print(response)
|
||||
print(response.usage)
|
||||
print(response.usage.completion_tokens)
|
||||
print(response["usage"]["completion_tokens"])
|
||||
# print("new cost tracking")
|
||||
except Exception as e:
|
||||
pytest.fail(f"Error occurred: {e}")
|
||||
|
||||
|
||||
# def test_completion_oobabooga():
|
||||
# try:
|
||||
# response = completion(
|
||||
|
|
|
|||
|
|
@ -219,3 +219,94 @@ def test_parallel_function_call_stream():
|
|||
|
||||
|
||||
# test_parallel_function_call_stream()
|
||||
|
||||
|
||||
def test_groq_parallel_function_call():
|
||||
litellm.set_verbose = True
|
||||
try:
|
||||
# Step 1: send the conversation and available functions to the model
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a function calling LLM that uses the data extracted from get_current_weather to answer questions about the weather in San Francisco.",
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's the weather like in San Francisco?",
|
||||
},
|
||||
]
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_current_weather",
|
||||
"description": "Get the current weather in a given location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {
|
||||
"type": "string",
|
||||
"description": "The city and state, e.g. San Francisco, CA",
|
||||
},
|
||||
"unit": {
|
||||
"type": "string",
|
||||
"enum": ["celsius", "fahrenheit"],
|
||||
},
|
||||
},
|
||||
"required": ["location"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
response = litellm.completion(
|
||||
model="groq/llama2-70b-4096",
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
tool_choice="auto", # auto is default, but we'll be explicit
|
||||
)
|
||||
print("Response\n", response)
|
||||
response_message = response.choices[0].message
|
||||
tool_calls = response_message.tool_calls
|
||||
|
||||
assert isinstance(response.choices[0].message.tool_calls[0].function.name, str)
|
||||
assert isinstance(
|
||||
response.choices[0].message.tool_calls[0].function.arguments, str
|
||||
)
|
||||
|
||||
print("length of tool calls", len(tool_calls))
|
||||
|
||||
# Step 2: check if the model wanted to call a function
|
||||
if tool_calls:
|
||||
# Step 3: call the function
|
||||
# Note: the JSON response may not always be valid; be sure to handle errors
|
||||
available_functions = {
|
||||
"get_current_weather": get_current_weather,
|
||||
} # only one function in this example, but you can have multiple
|
||||
messages.append(
|
||||
response_message
|
||||
) # extend conversation with assistant's reply
|
||||
print("Response message\n", response_message)
|
||||
# Step 4: send the info for each function call and function response to the model
|
||||
for tool_call in tool_calls:
|
||||
function_name = tool_call.function.name
|
||||
function_to_call = available_functions[function_name]
|
||||
function_args = json.loads(tool_call.function.arguments)
|
||||
function_response = function_to_call(
|
||||
location=function_args.get("location"),
|
||||
unit=function_args.get("unit"),
|
||||
)
|
||||
messages.append(
|
||||
{
|
||||
"tool_call_id": tool_call.id,
|
||||
"role": "tool",
|
||||
"name": function_name,
|
||||
"content": function_response,
|
||||
}
|
||||
) # extend conversation with function response
|
||||
print(f"messages: {messages}")
|
||||
second_response = litellm.completion(
|
||||
model="groq/llama2-70b-4096", messages=messages
|
||||
) # get a new response from the model where it can see the function response
|
||||
print("second response\n", second_response)
|
||||
except Exception as e:
|
||||
pytest.fail(f"Error occurred: {e}")
|
||||
|
|
|
|||
207
litellm/tests/test_prometheus_service.py
Normal file
207
litellm/tests/test_prometheus_service.py
Normal file
|
|
@ -0,0 +1,207 @@
|
|||
# What is this?
|
||||
## Unit Tests for prometheus service monitoring
|
||||
|
||||
import json
|
||||
import sys
|
||||
import os
|
||||
import io, asyncio
|
||||
|
||||
sys.path.insert(0, os.path.abspath("../.."))
|
||||
import pytest
|
||||
from litellm import acompletion, Cache
|
||||
from litellm._service_logger import ServiceLogging
|
||||
from litellm.integrations.prometheus_services import PrometheusServicesLogger
|
||||
import litellm
|
||||
|
||||
"""
|
||||
- Check if it receives a call when redis is used
|
||||
- Check if it fires messages accordingly
|
||||
"""
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_init_prometheus():
|
||||
"""
|
||||
- Run completion with caching
|
||||
- Assert success callback gets called
|
||||
"""
|
||||
|
||||
pl = PrometheusServicesLogger(mock_testing=True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_with_caching():
|
||||
"""
|
||||
- Run completion with caching
|
||||
- Assert success callback gets called
|
||||
"""
|
||||
|
||||
litellm.set_verbose = True
|
||||
litellm.cache = Cache(type="redis")
|
||||
litellm.service_callback = ["prometheus_system"]
|
||||
|
||||
sl = ServiceLogging(mock_testing=True)
|
||||
sl.prometheusServicesLogger.mock_testing = True
|
||||
litellm.cache.cache.service_logger_obj = sl
|
||||
|
||||
messages = [{"role": "user", "content": "Hey, how's it going?"}]
|
||||
response1 = await acompletion(
|
||||
model="gpt-3.5-turbo", messages=messages, caching=True
|
||||
)
|
||||
response1 = await acompletion(
|
||||
model="gpt-3.5-turbo", messages=messages, caching=True
|
||||
)
|
||||
|
||||
assert sl.mock_testing_async_success_hook > 0
|
||||
assert sl.prometheusServicesLogger.mock_testing_success_calls > 0
|
||||
assert sl.mock_testing_sync_failure_hook == 0
|
||||
assert sl.mock_testing_async_failure_hook == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_with_caching_bad_call():
|
||||
"""
|
||||
- Run completion with caching (incorrect credentials)
|
||||
- Assert failure callback gets called
|
||||
"""
|
||||
litellm.set_verbose = True
|
||||
sl = ServiceLogging(mock_testing=True)
|
||||
try:
|
||||
litellm.cache = Cache(type="redis", host="hello-world")
|
||||
litellm.service_callback = ["prometheus_system"]
|
||||
|
||||
litellm.cache.cache.service_logger_obj = sl
|
||||
|
||||
messages = [{"role": "user", "content": "Hey, how's it going?"}]
|
||||
response1 = await acompletion(
|
||||
model="gpt-3.5-turbo", messages=messages, caching=True
|
||||
)
|
||||
response1 = await acompletion(
|
||||
model="gpt-3.5-turbo", messages=messages, caching=True
|
||||
)
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
assert sl.mock_testing_async_failure_hook > 0
|
||||
assert sl.mock_testing_async_success_hook == 0
|
||||
assert sl.mock_testing_sync_success_hook == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_with_caching():
|
||||
"""
|
||||
- Run router with usage-based-routing-v2
|
||||
- Assert success callback gets called
|
||||
"""
|
||||
try:
|
||||
|
||||
def get_azure_params(deployment_name: str):
|
||||
params = {
|
||||
"model": f"azure/{deployment_name}",
|
||||
"api_key": os.environ["AZURE_API_KEY"],
|
||||
"api_version": os.environ["AZURE_API_VERSION"],
|
||||
"api_base": os.environ["AZURE_API_BASE"],
|
||||
}
|
||||
return params
|
||||
|
||||
model_list = [
|
||||
{
|
||||
"model_name": "azure/gpt-4",
|
||||
"litellm_params": get_azure_params("chatgpt-v-2"),
|
||||
"tpm": 100,
|
||||
},
|
||||
{
|
||||
"model_name": "azure/gpt-4",
|
||||
"litellm_params": get_azure_params("chatgpt-v-2"),
|
||||
"tpm": 1000,
|
||||
},
|
||||
]
|
||||
|
||||
router = litellm.Router(
|
||||
model_list=model_list,
|
||||
set_verbose=True,
|
||||
debug_level="DEBUG",
|
||||
routing_strategy="usage-based-routing-v2",
|
||||
redis_host=os.environ["REDIS_HOST"],
|
||||
redis_port=os.environ["REDIS_PORT"],
|
||||
redis_password=os.environ["REDIS_PASSWORD"],
|
||||
)
|
||||
|
||||
litellm.service_callback = ["prometheus_system"]
|
||||
|
||||
sl = ServiceLogging(mock_testing=True)
|
||||
sl.prometheusServicesLogger.mock_testing = True
|
||||
router.cache.redis_cache.service_logger_obj = sl
|
||||
|
||||
messages = [{"role": "user", "content": "Hey, how's it going?"}]
|
||||
response1 = await router.acompletion(model="azure/gpt-4", messages=messages)
|
||||
response1 = await router.acompletion(model="azure/gpt-4", messages=messages)
|
||||
|
||||
assert sl.mock_testing_async_success_hook > 0
|
||||
assert sl.mock_testing_sync_failure_hook == 0
|
||||
assert sl.mock_testing_async_failure_hook == 0
|
||||
assert sl.prometheusServicesLogger.mock_testing_success_calls > 0
|
||||
|
||||
except Exception as e:
|
||||
pytest.fail(f"An exception occured - {str(e)}")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_with_caching_bad_call():
|
||||
"""
|
||||
- Run completion with caching (incorrect credentials)
|
||||
- Assert failure callback gets called
|
||||
"""
|
||||
try:
|
||||
|
||||
def get_azure_params(deployment_name: str):
|
||||
params = {
|
||||
"model": f"azure/{deployment_name}",
|
||||
"api_key": os.environ["AZURE_API_KEY"],
|
||||
"api_version": os.environ["AZURE_API_VERSION"],
|
||||
"api_base": os.environ["AZURE_API_BASE"],
|
||||
}
|
||||
return params
|
||||
|
||||
model_list = [
|
||||
{
|
||||
"model_name": "azure/gpt-4",
|
||||
"litellm_params": get_azure_params("chatgpt-v-2"),
|
||||
"tpm": 100,
|
||||
},
|
||||
{
|
||||
"model_name": "azure/gpt-4",
|
||||
"litellm_params": get_azure_params("chatgpt-v-2"),
|
||||
"tpm": 1000,
|
||||
},
|
||||
]
|
||||
|
||||
router = litellm.Router(
|
||||
model_list=model_list,
|
||||
set_verbose=True,
|
||||
debug_level="DEBUG",
|
||||
routing_strategy="usage-based-routing-v2",
|
||||
redis_host="hello world",
|
||||
redis_port=os.environ["REDIS_PORT"],
|
||||
redis_password=os.environ["REDIS_PASSWORD"],
|
||||
)
|
||||
|
||||
litellm.service_callback = ["prometheus_system"]
|
||||
|
||||
sl = ServiceLogging(mock_testing=True)
|
||||
sl.prometheusServicesLogger.mock_testing = True
|
||||
router.cache.redis_cache.service_logger_obj = sl
|
||||
|
||||
messages = [{"role": "user", "content": "Hey, how's it going?"}]
|
||||
try:
|
||||
response1 = await router.acompletion(model="azure/gpt-4", messages=messages)
|
||||
response1 = await router.acompletion(model="azure/gpt-4", messages=messages)
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
assert sl.mock_testing_async_failure_hook > 0
|
||||
assert sl.mock_testing_async_success_hook == 0
|
||||
assert sl.mock_testing_sync_success_hook == 0
|
||||
|
||||
except Exception as e:
|
||||
pytest.fail(f"An exception occured - {str(e)}")
|
||||
|
|
@ -380,6 +380,51 @@ def test_completion_claude_stream():
|
|||
|
||||
|
||||
# test_completion_claude_stream()
|
||||
def test_completion_claude_2_stream():
|
||||
litellm.set_verbose = True
|
||||
response = completion(
|
||||
model="claude-2",
|
||||
messages=[{"role": "user", "content": "hello from litellm"}],
|
||||
stream=True,
|
||||
)
|
||||
complete_response = ""
|
||||
# Add any assertions here to check the response
|
||||
idx = 0
|
||||
for chunk in response:
|
||||
print(chunk)
|
||||
# print(chunk.choices[0].delta)
|
||||
chunk, finished = streaming_format_tests(idx, chunk)
|
||||
if finished:
|
||||
break
|
||||
complete_response += chunk
|
||||
idx += 1
|
||||
if complete_response.strip() == "":
|
||||
raise Exception("Empty response received")
|
||||
print(f"completion_response: {complete_response}")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_acompletion_claude_2_stream():
|
||||
litellm.set_verbose = True
|
||||
response = await litellm.acompletion(
|
||||
model="claude-2",
|
||||
messages=[{"role": "user", "content": "hello from litellm"}],
|
||||
stream=True,
|
||||
)
|
||||
complete_response = ""
|
||||
# Add any assertions here to check the response
|
||||
idx = 0
|
||||
async for chunk in response:
|
||||
print(chunk)
|
||||
# print(chunk.choices[0].delta)
|
||||
chunk, finished = streaming_format_tests(idx, chunk)
|
||||
if finished:
|
||||
break
|
||||
complete_response += chunk
|
||||
idx += 1
|
||||
if complete_response.strip() == "":
|
||||
raise Exception("Empty response received")
|
||||
print(f"completion_response: {complete_response}")
|
||||
|
||||
|
||||
def test_completion_palm_stream():
|
||||
|
|
|
|||
|
|
@ -219,17 +219,19 @@ def test_validate_environment_empty_model():
|
|||
@mock.patch.dict(os.environ, {"OLLAMA_API_BASE": "foo"}, clear=True)
|
||||
def test_validate_environment_ollama():
|
||||
for provider in ["ollama", "ollama_chat"]:
|
||||
kv = validate_environment(provider+"/mistral")
|
||||
kv = validate_environment(provider + "/mistral")
|
||||
assert kv["keys_in_environment"]
|
||||
assert kv["missing_keys"] == []
|
||||
|
||||
|
||||
@mock.patch.dict(os.environ, {}, clear=True)
|
||||
def test_validate_environment_ollama_failed():
|
||||
for provider in ["ollama", "ollama_chat"]:
|
||||
kv = validate_environment(provider+"/mistral")
|
||||
kv = validate_environment(provider + "/mistral")
|
||||
assert not kv["keys_in_environment"]
|
||||
assert kv["missing_keys"] == ["OLLAMA_API_BASE"]
|
||||
|
||||
|
||||
def test_function_to_dict():
|
||||
print("testing function to dict for get current weather")
|
||||
|
||||
|
|
@ -338,6 +340,7 @@ def test_supports_function_calling():
|
|||
assert (
|
||||
litellm.supports_function_calling(model="azure/gpt-4-1106-preview") == True
|
||||
)
|
||||
assert litellm.supports_function_calling(model="groq/gemma-7b-it") == True
|
||||
assert (
|
||||
litellm.supports_function_calling(model="anthropic.claude-instant-v1")
|
||||
== False
|
||||
|
|
|
|||
30
litellm/types/services.py
Normal file
30
litellm/types/services.py
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
import uuid, enum
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ServiceTypes(enum.Enum):
|
||||
"""
|
||||
Enum for litellm-adjacent services (redis/postgres/etc.)
|
||||
"""
|
||||
|
||||
REDIS = "redis"
|
||||
DB = "postgres"
|
||||
|
||||
|
||||
class ServiceLoggerPayload(BaseModel):
|
||||
"""
|
||||
The payload logged during service success/failure
|
||||
"""
|
||||
|
||||
is_error: bool = Field(description="did an error occur")
|
||||
error: Optional[str] = Field(None, description="what was the error")
|
||||
service: ServiceTypes = Field(description="who is this for? - postgres/redis")
|
||||
duration: float = Field(description="How long did the request take?")
|
||||
|
||||
def to_json(self, **kwargs):
|
||||
try:
|
||||
return self.model_dump(**kwargs) # noqa
|
||||
except Exception as e:
|
||||
# if using pydantic v1
|
||||
return self.dict(**kwargs)
|
||||
|
|
@ -29,7 +29,9 @@ from tokenizers import Tokenizer
|
|||
from dataclasses import (
|
||||
dataclass,
|
||||
field,
|
||||
) # for storing API inputs, outputs, and metadata
|
||||
)
|
||||
|
||||
import litellm._service_logger # for storing API inputs, outputs, and metadata
|
||||
|
||||
try:
|
||||
# this works in python 3.8
|
||||
|
|
@ -69,6 +71,7 @@ from .integrations.custom_logger import CustomLogger
|
|||
from .integrations.langfuse import LangFuseLogger
|
||||
from .integrations.datadog import DataDogLogger
|
||||
from .integrations.prometheus import PrometheusLogger
|
||||
from .integrations.prometheus_services import PrometheusServicesLogger
|
||||
from .integrations.dynamodb import DyanmoDBLogger
|
||||
from .integrations.s3 import S3Logger
|
||||
from .integrations.clickhouse import ClickhouseLogger
|
||||
|
|
@ -4520,6 +4523,7 @@ def get_optional_params(
|
|||
and custom_llm_provider != "vertex_ai"
|
||||
and custom_llm_provider != "anyscale"
|
||||
and custom_llm_provider != "together_ai"
|
||||
and custom_llm_provider != "groq"
|
||||
and custom_llm_provider != "mistral"
|
||||
and custom_llm_provider != "anthropic"
|
||||
and custom_llm_provider != "cohere_chat"
|
||||
|
|
@ -5219,6 +5223,29 @@ def get_optional_params(
|
|||
optional_params["extra_body"] = (
|
||||
extra_body # openai client supports `extra_body` param
|
||||
)
|
||||
elif custom_llm_provider == "groq":
|
||||
supported_params = get_supported_openai_params(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
_check_valid_arg(supported_params=supported_params)
|
||||
|
||||
if temperature is not None:
|
||||
optional_params["temperature"] = temperature
|
||||
if max_tokens is not None:
|
||||
optional_params["max_tokens"] = max_tokens
|
||||
if top_p is not None:
|
||||
optional_params["top_p"] = top_p
|
||||
if stream is not None:
|
||||
optional_params["stream"] = stream
|
||||
if stop is not None:
|
||||
optional_params["stop"] = stop
|
||||
if tools is not None:
|
||||
optional_params["tools"] = tools
|
||||
if tool_choice is not None:
|
||||
optional_params["tool_choice"] = tool_choice
|
||||
if response_format is not None:
|
||||
optional_params["response_format"] = tool_choice
|
||||
|
||||
elif custom_llm_provider == "openrouter":
|
||||
supported_params = get_supported_openai_params(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
|
|
@ -5423,6 +5450,17 @@ def get_supported_openai_params(model: str, custom_llm_provider: str):
|
|||
"tools",
|
||||
"tool_choice",
|
||||
]
|
||||
elif custom_llm_provider == "groq":
|
||||
return [
|
||||
"temperature",
|
||||
"max_tokens",
|
||||
"top_p",
|
||||
"stream",
|
||||
"stop",
|
||||
"tools",
|
||||
"tool_choice",
|
||||
"response_format",
|
||||
]
|
||||
elif custom_llm_provider == "cohere":
|
||||
return [
|
||||
"stream",
|
||||
|
|
@ -6564,7 +6602,9 @@ def handle_failure(exception, traceback_exception, start_time, end_time, args, k
|
|||
for detail in additional_details:
|
||||
slack_msg += f"{detail}: {additional_details[detail]}\n"
|
||||
slack_msg += f"Traceback: {traceback_exception}"
|
||||
truncated_slack_msg = textwrap.shorten(slack_msg, width=512, placeholder="...")
|
||||
truncated_slack_msg = textwrap.shorten(
|
||||
slack_msg, width=512, placeholder="..."
|
||||
)
|
||||
slack_app.client.chat_postMessage(
|
||||
channel=alerts_channel, text=truncated_slack_msg
|
||||
)
|
||||
|
|
@ -7494,7 +7534,6 @@ def exception_type(
|
|||
message=f"AnthropicException - {original_exception.message}",
|
||||
model=model,
|
||||
llm_provider="anthropic",
|
||||
request=original_exception.request,
|
||||
)
|
||||
elif original_exception.status_code == 429:
|
||||
exception_mapping_worked = True
|
||||
|
|
@ -7591,7 +7630,6 @@ def exception_type(
|
|||
message=f"ReplicateException - {original_exception.message}",
|
||||
model=model,
|
||||
llm_provider="replicate",
|
||||
request=original_exception.request,
|
||||
)
|
||||
elif original_exception.status_code == 429:
|
||||
exception_mapping_worked = True
|
||||
|
|
@ -8000,7 +8038,6 @@ def exception_type(
|
|||
message=f"HuggingfaceException - {original_exception.message}",
|
||||
model=model,
|
||||
llm_provider="huggingface",
|
||||
request=original_exception.request,
|
||||
)
|
||||
elif original_exception.status_code == 429:
|
||||
exception_mapping_worked = True
|
||||
|
|
@ -8060,7 +8097,6 @@ def exception_type(
|
|||
message=f"AI21Exception - {original_exception.message}",
|
||||
model=model,
|
||||
llm_provider="ai21",
|
||||
request=original_exception.request,
|
||||
)
|
||||
if original_exception.status_code == 422:
|
||||
exception_mapping_worked = True
|
||||
|
|
@ -8150,7 +8186,6 @@ def exception_type(
|
|||
message=f"NLPCloudException - {original_exception.message}",
|
||||
model=model,
|
||||
llm_provider="nlp_cloud",
|
||||
request=original_exception.request,
|
||||
)
|
||||
elif (
|
||||
original_exception.status_code == 429
|
||||
|
|
@ -8267,7 +8302,6 @@ def exception_type(
|
|||
message=f"TogetherAIException - {original_exception.message}",
|
||||
model=model,
|
||||
llm_provider="together_ai",
|
||||
request=original_exception.request,
|
||||
)
|
||||
elif original_exception.status_code == 422:
|
||||
exception_mapping_worked = True
|
||||
|
|
@ -8467,7 +8501,6 @@ def exception_type(
|
|||
message=f"AzureException - {original_exception.message}",
|
||||
model=model,
|
||||
llm_provider="azure",
|
||||
request=original_exception.request,
|
||||
)
|
||||
if original_exception.status_code == 422:
|
||||
exception_mapping_worked = True
|
||||
|
|
@ -8805,6 +8838,35 @@ class CustomStreamWrapper:
|
|||
self.holding_chunk = ""
|
||||
return hold, curr_chunk
|
||||
|
||||
def handle_anthropic_text_chunk(self, chunk):
|
||||
str_line = chunk
|
||||
if isinstance(chunk, bytes): # Handle binary data
|
||||
str_line = chunk.decode("utf-8") # Convert bytes to string
|
||||
text = ""
|
||||
is_finished = False
|
||||
finish_reason = None
|
||||
if str_line.startswith("data:"):
|
||||
data_json = json.loads(str_line[5:])
|
||||
type_chunk = data_json.get("type", None)
|
||||
if type_chunk == "completion":
|
||||
text = data_json.get("completion")
|
||||
finish_reason = data_json.get("stop_reason")
|
||||
if finish_reason is not None:
|
||||
is_finished = True
|
||||
return {
|
||||
"text": text,
|
||||
"is_finished": is_finished,
|
||||
"finish_reason": finish_reason,
|
||||
}
|
||||
elif "error" in str_line:
|
||||
raise ValueError(f"Unable to parse response. Original response: {str_line}")
|
||||
else:
|
||||
return {
|
||||
"text": text,
|
||||
"is_finished": is_finished,
|
||||
"finish_reason": finish_reason,
|
||||
}
|
||||
|
||||
def handle_anthropic_chunk(self, chunk):
|
||||
str_line = chunk
|
||||
if isinstance(chunk, bytes): # Handle binary data
|
||||
|
|
@ -9492,6 +9554,14 @@ class CustomStreamWrapper:
|
|||
completion_obj["content"] = response_obj["text"]
|
||||
if response_obj["is_finished"]:
|
||||
self.received_finish_reason = response_obj["finish_reason"]
|
||||
elif (
|
||||
self.custom_llm_provider
|
||||
and self.custom_llm_provider == "anthropic_text"
|
||||
):
|
||||
response_obj = self.handle_anthropic_text_chunk(chunk)
|
||||
completion_obj["content"] = response_obj["text"]
|
||||
if response_obj["is_finished"]:
|
||||
self.received_finish_reason = response_obj["finish_reason"]
|
||||
elif self.model == "replicate" or self.custom_llm_provider == "replicate":
|
||||
response_obj = self.handle_replicate_chunk(chunk)
|
||||
completion_obj["content"] = response_obj["text"]
|
||||
|
|
@ -10069,6 +10139,7 @@ class CustomStreamWrapper:
|
|||
or self.custom_llm_provider == "text-completion-openai"
|
||||
or self.custom_llm_provider == "azure_text"
|
||||
or self.custom_llm_provider == "anthropic"
|
||||
or self.custom_llm_provider == "anthropic_text"
|
||||
or self.custom_llm_provider == "huggingface"
|
||||
or self.custom_llm_provider == "ollama"
|
||||
or self.custom_llm_provider == "ollama_chat"
|
||||
|
|
|
|||
|
|
@ -714,7 +714,8 @@
|
|||
"input_cost_per_token": 0.00000070,
|
||||
"output_cost_per_token": 0.00000080,
|
||||
"litellm_provider": "groq",
|
||||
"mode": "chat"
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true
|
||||
},
|
||||
"groq/mixtral-8x7b-32768": {
|
||||
"max_tokens": 32768,
|
||||
|
|
@ -723,7 +724,8 @@
|
|||
"input_cost_per_token": 0.00000027,
|
||||
"output_cost_per_token": 0.00000027,
|
||||
"litellm_provider": "groq",
|
||||
"mode": "chat"
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true
|
||||
},
|
||||
"groq/gemma-7b-it": {
|
||||
"max_tokens": 8192,
|
||||
|
|
@ -732,7 +734,8 @@
|
|||
"input_cost_per_token": 0.00000010,
|
||||
"output_cost_per_token": 0.00000010,
|
||||
"litellm_provider": "groq",
|
||||
"mode": "chat"
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true
|
||||
},
|
||||
"claude-instant-1.2": {
|
||||
"max_tokens": 8191,
|
||||
|
|
@ -1079,7 +1082,8 @@
|
|||
"source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models"
|
||||
},
|
||||
"vertex_ai/claude-3-sonnet@20240229": {
|
||||
"max_tokens": 200000,
|
||||
"max_tokens": 4096,
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 4096,
|
||||
"input_cost_per_token": 0.000003,
|
||||
"output_cost_per_token": 0.000015,
|
||||
|
|
@ -1087,13 +1091,23 @@
|
|||
"mode": "chat"
|
||||
},
|
||||
"vertex_ai/claude-3-haiku@20240307": {
|
||||
"max_tokens": 200000,
|
||||
"max_tokens": 4096,
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 4096,
|
||||
"input_cost_per_token": 0.00000025,
|
||||
"output_cost_per_token": 0.00000125,
|
||||
"litellm_provider": "vertex_ai-anthropic_models",
|
||||
"mode": "chat"
|
||||
},
|
||||
"vertex_ai/claude-3-opus@20240229": {
|
||||
"max_tokens": 4096,
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 4096,
|
||||
"input_cost_per_token": 0.0000015,
|
||||
"output_cost_per_token": 0.0000075,
|
||||
"litellm_provider": "vertex_ai-anthropic_models",
|
||||
"mode": "chat"
|
||||
},
|
||||
"textembedding-gecko": {
|
||||
"max_tokens": 3072,
|
||||
"max_input_tokens": 3072,
|
||||
|
|
|
|||
|
|
@ -55,6 +55,10 @@ model_list:
|
|||
api_base: https://openai-function-calling-workers.tasslexyz.workers.dev/
|
||||
stream_timeout: 0.001
|
||||
rpm: 1
|
||||
- model_name: "*"
|
||||
litellm_params:
|
||||
model: openai/*
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
- model_name: gpt-instruct # [PROD TEST] - tests if `/health` automatically infers this to be a text completion model
|
||||
litellm_params:
|
||||
model: text-completion-openai/gpt-3.5-turbo-instruct
|
||||
|
|
@ -66,6 +70,15 @@ litellm_settings:
|
|||
request_timeout: 600
|
||||
telemetry: False
|
||||
context_window_fallbacks: [{"gpt-3.5-turbo": ["gpt-3.5-turbo-large"]}]
|
||||
default_team_settings:
|
||||
- team_id: team-1
|
||||
success_callback: ["langfuse"]
|
||||
langfuse_public_key: os.environ/LANGFUSE_PROJECT1_PUBLIC # Project 1
|
||||
langfuse_secret: os.environ/LANGFUSE_PROJECT1_SECRET # Project 1
|
||||
- team_id: team-2
|
||||
success_callback: ["langfuse"]
|
||||
langfuse_public_key: os.environ/LANGFUSE_PROJECT2_PUBLIC # Project 2
|
||||
langfuse_secret: os.environ/LANGFUSE_PROJECT2_SECRET # Project 2
|
||||
|
||||
router_settings:
|
||||
routing_strategy: usage-based-routing-v2
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[tool.poetry]
|
||||
name = "litellm"
|
||||
version = "1.35.4"
|
||||
version = "1.35.6"
|
||||
description = "Library to easily interface with LLM API providers"
|
||||
authors = ["BerriAI"]
|
||||
license = "MIT"
|
||||
|
|
@ -80,7 +80,7 @@ requires = ["poetry-core", "wheel"]
|
|||
build-backend = "poetry.core.masonry.api"
|
||||
|
||||
[tool.commitizen]
|
||||
version = "1.35.4"
|
||||
version = "1.35.6"
|
||||
version_files = [
|
||||
"pyproject.toml:^version"
|
||||
]
|
||||
|
|
|
|||
|
|
@ -4,6 +4,11 @@
|
|||
import pytest
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import os
|
||||
import dotenv
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
|
||||
async def generate_key(session, models=[]):
|
||||
|
|
@ -36,7 +41,7 @@ async def get_models(session, key):
|
|||
async with session.get(url, headers=headers) as response:
|
||||
status = response.status
|
||||
response_text = await response.text()
|
||||
|
||||
print("response from /models")
|
||||
print(response_text)
|
||||
print()
|
||||
|
||||
|
|
@ -102,14 +107,14 @@ async def get_model_info(session, key):
|
|||
return await response.json()
|
||||
|
||||
|
||||
async def chat_completion(session, key):
|
||||
async def chat_completion(session, key, model="azure-gpt-3.5"):
|
||||
url = "http://0.0.0.0:4000/chat/completions"
|
||||
headers = {
|
||||
"Authorization": f"Bearer {key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
data = {
|
||||
"model": "azure-gpt-3.5",
|
||||
"model": model,
|
||||
"messages": [
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": "Hello!"},
|
||||
|
|
@ -177,3 +182,112 @@ async def test_add_and_delete_models():
|
|||
await asyncio.sleep(60)
|
||||
await chat_completion(session=session, key=key)
|
||||
await delete_model(session=session, model_id=model_id)
|
||||
|
||||
|
||||
async def add_model_for_health_checking(session, model_id="123"):
|
||||
url = "http://0.0.0.0:4000/model/new"
|
||||
headers = {
|
||||
"Authorization": f"Bearer sk-1234",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
data = {
|
||||
"model_name": f"azure-model-health-check-{model_id}",
|
||||
"litellm_params": {
|
||||
"model": "azure/chatgpt-v-2",
|
||||
"api_key": os.getenv("AZURE_API_KEY"),
|
||||
"api_base": "https://openai-gpt-4-test-v-1.openai.azure.com/",
|
||||
"api_version": "2023-05-15",
|
||||
},
|
||||
"model_info": {"id": model_id},
|
||||
}
|
||||
|
||||
async with session.post(url, headers=headers, json=data) as response:
|
||||
status = response.status
|
||||
response_text = await response.text()
|
||||
|
||||
print(f"Add models {response_text}")
|
||||
print()
|
||||
|
||||
if status != 200:
|
||||
raise Exception(f"Request did not return a 200 status code: {status}")
|
||||
|
||||
|
||||
async def get_model_info_v2(session, key):
|
||||
url = "http://0.0.0.0:4000/v2/model/info"
|
||||
headers = {
|
||||
"Authorization": f"Bearer {key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
async with session.get(url, headers=headers) as response:
|
||||
status = response.status
|
||||
response_text = await response.text()
|
||||
print("response from v2/model/info")
|
||||
print(response_text)
|
||||
print()
|
||||
|
||||
if status != 200:
|
||||
raise Exception(f"Request did not return a 200 status code: {status}")
|
||||
|
||||
|
||||
async def get_model_health(session, key, model_name):
|
||||
url = "http://0.0.0.0:4000/health?model=" + model_name
|
||||
headers = {
|
||||
"Authorization": f"Bearer {key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
async with session.get(url, headers=headers) as response:
|
||||
status = response.status
|
||||
response_text = await response.json()
|
||||
print("response from /health?model=", model_name)
|
||||
print(response_text)
|
||||
print()
|
||||
|
||||
if status != 200:
|
||||
raise Exception(f"Request did not return a 200 status code: {status}")
|
||||
return response_text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_model_run_health():
|
||||
"""
|
||||
Add model
|
||||
Call /model/info and v2/model/info
|
||||
-> Admin UI calls v2/model/info
|
||||
Call /chat/completions
|
||||
Call /health
|
||||
-> Ensure the health check for the endpoint is working as expected
|
||||
"""
|
||||
import uuid
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
key_gen = await generate_key(session=session)
|
||||
key = key_gen["key"]
|
||||
model_id = str(uuid.uuid4())
|
||||
model_name = f"azure-model-health-check-{model_id}"
|
||||
print("adding model", model_name)
|
||||
await add_model_for_health_checking(session=session, model_id=model_id)
|
||||
await asyncio.sleep(30)
|
||||
print("calling /model/info")
|
||||
await get_model_info(session=session, key=key)
|
||||
print("calling v2/model/info")
|
||||
await get_model_info_v2(session=session, key=key)
|
||||
|
||||
print("calling /chat/completions -> expect to work")
|
||||
await chat_completion(session=session, key=key, model=model_name)
|
||||
|
||||
print("calling /health?model=", model_name)
|
||||
_health_info = await get_model_health(
|
||||
session=session, key=key, model_name=model_name
|
||||
)
|
||||
_healthy_endpooint = _health_info["healthy_endpoints"][0]
|
||||
|
||||
assert _health_info["healthy_count"] == 1
|
||||
assert (
|
||||
_healthy_endpooint["model"] == "azure/chatgpt-v-2"
|
||||
) # this is the model that got added
|
||||
|
||||
# cleanup
|
||||
await delete_model(session=session, model_id=model_id)
|
||||
|
|
|
|||
|
|
@ -14,16 +14,19 @@ def response_header_check(response):
|
|||
assert headers_size < 4096, "Response headers exceed the 4kb limit"
|
||||
|
||||
|
||||
async def generate_key(session):
|
||||
async def generate_key(
|
||||
session,
|
||||
models=[
|
||||
"gpt-4",
|
||||
"text-embedding-ada-002",
|
||||
"dall-e-2",
|
||||
"fake-openai-endpoint-2",
|
||||
],
|
||||
):
|
||||
url = "http://0.0.0.0:4000/key/generate"
|
||||
headers = {"Authorization": "Bearer sk-1234", "Content-Type": "application/json"}
|
||||
data = {
|
||||
"models": [
|
||||
"gpt-4",
|
||||
"text-embedding-ada-002",
|
||||
"dall-e-2",
|
||||
"fake-openai-endpoint-2",
|
||||
],
|
||||
"models": models,
|
||||
"duration": None,
|
||||
}
|
||||
|
||||
|
|
@ -294,3 +297,19 @@ async def test_image_generation():
|
|||
key_gen = await new_user(session=session)
|
||||
key_2 = key_gen["key"]
|
||||
await image_generation(session=session, key=key_2)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_wildcard_chat_completion():
|
||||
"""
|
||||
- Create key for model = "*" -> this has access to all models
|
||||
- proxy_server_config.yaml has model = *
|
||||
- Make chat completion call
|
||||
|
||||
"""
|
||||
async with aiohttp.ClientSession() as session:
|
||||
key_gen = await generate_key(session=session, models=["*"])
|
||||
key = key_gen["key"]
|
||||
|
||||
# call chat/completions with a model that the key was not created for + the model is not on the config.yaml
|
||||
await chat_completion(session=session, key=key, model="gpt-3.5-turbo-0125")
|
||||
|
|
|
|||
|
|
@ -37,6 +37,29 @@ async def new_user(
|
|||
return await response.json()
|
||||
|
||||
|
||||
async def add_member(session, i, team_id, user_id=None, user_email=None):
|
||||
url = "http://0.0.0.0:4000/team/member_add"
|
||||
headers = {"Authorization": "Bearer sk-1234", "Content-Type": "application/json"}
|
||||
data = {"team_id": team_id, "member": {"role": "user"}}
|
||||
if user_email is not None:
|
||||
data["member"]["user_email"] = user_email
|
||||
elif user_id is not None:
|
||||
data["member"]["user_id"] = user_id
|
||||
|
||||
async with session.post(url, headers=headers, json=data) as response:
|
||||
status = response.status
|
||||
response_text = await response.text()
|
||||
|
||||
print(f"ADD MEMBER Response {i} (Status code: {status}):")
|
||||
print(response_text)
|
||||
print()
|
||||
|
||||
if status != 200:
|
||||
raise Exception(f"Request {i} did not return a 200 status code: {status}")
|
||||
|
||||
return await response.json()
|
||||
|
||||
|
||||
async def delete_member(session, i, team_id, user_id):
|
||||
url = "http://0.0.0.0:4000/team/member_delete"
|
||||
headers = {"Authorization": "Bearer sk-1234", "Content-Type": "application/json"}
|
||||
|
|
@ -247,43 +270,75 @@ async def test_team_info():
|
|||
await get_team_info(session=session, get_team=team_id, call_key="sk-1234")
|
||||
|
||||
|
||||
"""
|
||||
- Create team
|
||||
- Add user (user exists in db)
|
||||
- Update team
|
||||
- Check if it works
|
||||
"""
|
||||
|
||||
"""
|
||||
- Create team
|
||||
- Add user (user doesn't exist in db)
|
||||
- Update team
|
||||
- Check if it works
|
||||
"""
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_update():
|
||||
async def test_team_update_sc_2():
|
||||
"""
|
||||
- Create team with 1 admin, 1 user
|
||||
- Create new user
|
||||
- Replace existing user with new user in team
|
||||
- Create team
|
||||
- Add 1 user (doesn't exist in db)
|
||||
- Change team alias
|
||||
- Check if it works
|
||||
- Assert team object unchanged besides team alias
|
||||
"""
|
||||
async with aiohttp.ClientSession() as session:
|
||||
## Create admin
|
||||
admin_user = f"{uuid.uuid4()}"
|
||||
await new_user(session=session, i=0, user_id=admin_user)
|
||||
## Create normal user
|
||||
normal_user = f"{uuid.uuid4()}"
|
||||
await new_user(session=session, i=0, user_id=normal_user)
|
||||
## Create team with 1 admin and 1 user
|
||||
member_list = [
|
||||
{"role": "admin", "user_id": admin_user},
|
||||
{"role": "user", "user_id": normal_user},
|
||||
]
|
||||
team_data = await new_team(session=session, i=0, member_list=member_list)
|
||||
## Create new normal user
|
||||
new_normal_user = f"{uuid.uuid4()}"
|
||||
await new_user(session=session, i=0, user_id=new_normal_user)
|
||||
## Update member list
|
||||
member_list = [
|
||||
{"role": "admin", "user_id": admin_user},
|
||||
{"role": "user", "user_id": new_normal_user},
|
||||
]
|
||||
team_data = await update_team(
|
||||
new_normal_user = f"krrish_{uuid.uuid4()}@berri.ai"
|
||||
await add_member(
|
||||
session=session,
|
||||
i=0,
|
||||
member_list=member_list,
|
||||
team_id=team_data["team_id"],
|
||||
tpm_limit=100,
|
||||
user_id=None,
|
||||
user_email=new_normal_user,
|
||||
)
|
||||
|
||||
assert team_data["data"]["tpm_limit"] == 100
|
||||
## CHANGE TEAM ALIAS
|
||||
|
||||
new_team_data = await update_team(
|
||||
session=session, i=0, team_id=team_data["team_id"], team_alias="my-new-team"
|
||||
)
|
||||
|
||||
assert new_team_data["data"]["team_alias"] == "my-new-team"
|
||||
print(f"team_data: {team_data}")
|
||||
## assert rest of object is the same
|
||||
for k, v in new_team_data["data"].items():
|
||||
if (
|
||||
k == "members_with_roles"
|
||||
): # assert 1 more member (role: "user", user_email: $user_email)
|
||||
len(new_team_data["data"][k]) == len(team_data[k]) + 1
|
||||
elif (
|
||||
k == "created_at"
|
||||
or k == "updated_at"
|
||||
or k == "model_spend"
|
||||
or k == "model_max_budget"
|
||||
or k == "model_id"
|
||||
or k == "litellm_organization_table"
|
||||
or k == "litellm_model_table"
|
||||
):
|
||||
pass
|
||||
else:
|
||||
assert new_team_data["data"][k] == team_data[k]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
173
tests/test_team_logging.py
Normal file
173
tests/test_team_logging.py
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
# What this tests ?
|
||||
## Tests /models and /model/* endpoints
|
||||
|
||||
import pytest
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import os
|
||||
import dotenv
|
||||
from dotenv import load_dotenv
|
||||
import pytest
|
||||
|
||||
load_dotenv()
|
||||
|
||||
|
||||
async def generate_key(session, models=[], team_id=None):
|
||||
url = "http://0.0.0.0:4000/key/generate"
|
||||
headers = {"Authorization": "Bearer sk-1234", "Content-Type": "application/json"}
|
||||
data = {
|
||||
"models": models,
|
||||
"duration": None,
|
||||
"team_id": team_id,
|
||||
}
|
||||
|
||||
async with session.post(url, headers=headers, json=data) as response:
|
||||
status = response.status
|
||||
response_text = await response.text()
|
||||
|
||||
print(response_text)
|
||||
print()
|
||||
|
||||
if status != 200:
|
||||
raise Exception(f"Request did not return a 200 status code: {status}")
|
||||
return await response.json()
|
||||
|
||||
|
||||
async def chat_completion(session, key, model="azure-gpt-3.5", request_metadata=None):
|
||||
url = "http://0.0.0.0:4000/chat/completions"
|
||||
headers = {
|
||||
"Authorization": f"Bearer {key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
data = {
|
||||
"model": model,
|
||||
"messages": [
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": "Hello!"},
|
||||
],
|
||||
"metadata": request_metadata,
|
||||
}
|
||||
|
||||
print("data sent in test=", data)
|
||||
|
||||
async with session.post(url, headers=headers, json=data) as response:
|
||||
status = response.status
|
||||
response_text = await response.text()
|
||||
|
||||
print(response_text)
|
||||
print()
|
||||
|
||||
if status != 200:
|
||||
raise Exception(f"Request did not return a 200 status code: {status}")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_logging():
|
||||
"""
|
||||
-> Team 1 logs to project 1
|
||||
-> Create Key
|
||||
-> Make chat/completions call
|
||||
-> Fetch logs from langfuse
|
||||
"""
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
|
||||
key = await generate_key(
|
||||
session, models=["fake-openai-endpoint"], team_id="team-1"
|
||||
) # team-1 logs to project 1
|
||||
|
||||
import uuid
|
||||
|
||||
_trace_id = f"trace-{uuid.uuid4()}"
|
||||
_request_metadata = {
|
||||
"trace_id": _trace_id,
|
||||
}
|
||||
|
||||
await chat_completion(
|
||||
session,
|
||||
key["key"],
|
||||
model="fake-openai-endpoint",
|
||||
request_metadata=_request_metadata,
|
||||
)
|
||||
|
||||
# Test - if the logs were sent to the correct team on langfuse
|
||||
import langfuse
|
||||
|
||||
langfuse_client = langfuse.Langfuse(
|
||||
public_key=os.getenv("LANGFUSE_PROJECT1_PUBLIC"),
|
||||
secret_key=os.getenv("LANGFUSE_PROJECT1_SECRET"),
|
||||
)
|
||||
|
||||
await asyncio.sleep(10)
|
||||
|
||||
print(f"searching for trace_id={_trace_id} on langfuse")
|
||||
|
||||
generations = langfuse_client.get_generations(trace_id=_trace_id).data
|
||||
print(generations)
|
||||
assert len(generations) == 1
|
||||
except Exception as e:
|
||||
pytest.fail(f"Unexpected error: {str(e)}")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_2logging():
|
||||
"""
|
||||
-> Team 1 logs to project 2
|
||||
-> Create Key
|
||||
-> Make chat/completions call
|
||||
-> Fetch logs from langfuse
|
||||
"""
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
|
||||
key = await generate_key(
|
||||
session, models=["fake-openai-endpoint"], team_id="team-2"
|
||||
) # team-1 logs to project 1
|
||||
|
||||
import uuid
|
||||
|
||||
_trace_id = f"trace-{uuid.uuid4()}"
|
||||
_request_metadata = {
|
||||
"trace_id": _trace_id,
|
||||
}
|
||||
|
||||
await chat_completion(
|
||||
session,
|
||||
key["key"],
|
||||
model="fake-openai-endpoint",
|
||||
request_metadata=_request_metadata,
|
||||
)
|
||||
|
||||
# Test - if the logs were sent to the correct team on langfuse
|
||||
import langfuse
|
||||
|
||||
langfuse_client = langfuse.Langfuse(
|
||||
public_key=os.getenv("LANGFUSE_PROJECT2_PUBLIC"),
|
||||
secret_key=os.getenv("LANGFUSE_PROJECT2_SECRET"),
|
||||
)
|
||||
|
||||
await asyncio.sleep(10)
|
||||
|
||||
print(f"searching for trace_id={_trace_id} on langfuse")
|
||||
|
||||
generations = langfuse_client.get_generations(trace_id=_trace_id).data
|
||||
print("Team 2 generations", generations)
|
||||
|
||||
# team-2 should have 1 generation with this trace id
|
||||
assert len(generations) == 1
|
||||
|
||||
# team-1 should have 0 generations with this trace id
|
||||
langfuse_client_1 = langfuse.Langfuse(
|
||||
public_key=os.getenv("LANGFUSE_PROJECT1_PUBLIC"),
|
||||
secret_key=os.getenv("LANGFUSE_PROJECT1_SECRET"),
|
||||
)
|
||||
|
||||
generations_team_1 = langfuse_client_1.get_generations(
|
||||
trace_id=_trace_id
|
||||
).data
|
||||
print("Team 1 generations", generations_team_1)
|
||||
|
||||
assert len(generations_team_1) == 0
|
||||
|
||||
except Exception as e:
|
||||
pytest.fail("Team 2 logging failed: " + str(e))
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1 +0,0 @@
|
|||
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[165],{83155:function(e,t,n){(window.__NEXT_P=window.__NEXT_P||[]).push(["/_not-found",function(){return n(84032)}])},84032:function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),Object.defineProperty(t,"default",{enumerable:!0,get:function(){return i}}),n(86921);let o=n(3827);n(64090);let r={error:{fontFamily:'system-ui,"Segoe UI",Roboto,Helvetica,Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji"',height:"100vh",textAlign:"center",display:"flex",flexDirection:"column",alignItems:"center",justifyContent:"center"},desc:{display:"inline-block"},h1:{display:"inline-block",margin:"0 20px 0 0",padding:"0 23px 0 0",fontSize:24,fontWeight:500,verticalAlign:"top",lineHeight:"49px"},h2:{fontSize:14,fontWeight:400,lineHeight:"49px",margin:0}};function i(){return(0,o.jsxs)(o.Fragment,{children:[(0,o.jsx)("title",{children:"404: This page could not be found."}),(0,o.jsx)("div",{style:r.error,children:(0,o.jsxs)("div",{children:[(0,o.jsx)("style",{dangerouslySetInnerHTML:{__html:"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}),(0,o.jsx)("h1",{className:"next-error-h1",style:r.h1,children:"404"}),(0,o.jsx)("div",{style:r.desc,children:(0,o.jsx)("h2",{style:r.h2,children:"This page could not be found."})})]})})]})}("function"==typeof t.default||"object"==typeof t.default&&null!==t.default)&&void 0===t.default.__esModule&&(Object.defineProperty(t.default,"__esModule",{value:!0}),Object.assign(t.default,t),e.exports=t.default)}},function(e){e.O(0,[971,69,744],function(){return e(e.s=83155)}),_N_E=e.O()}]);
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1 +1 @@
|
|||
<!DOCTYPE html><html id="__next_error__"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/ui/_next/static/chunks/webpack-11b043d6a7ef78fa.js" crossorigin=""/><script src="/ui/_next/static/chunks/fd9d1056-a507ee9e75a3be72.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/69-589b47e7a69d316f.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/main-app-096338c8e1915716.js" async="" crossorigin=""></script><title>LiteLLM Dashboard</title><meta name="description" content="LiteLLM Proxy Admin UI"/><link rel="icon" href="/ui/favicon.ico" type="image/x-icon" sizes="16x16"/><meta name="next-size-adjust"/><script src="/ui/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js" crossorigin="" noModule=""></script></head><body><script src="/ui/_next/static/chunks/webpack-11b043d6a7ef78fa.js" crossorigin="" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/ui/_next/static/media/c9a5bc6a7c948fb0-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n2:HL[\"/ui/_next/static/css/a282d1bfd6ed4df8.css\",\"style\",{\"crossOrigin\":\"\"}]\n0:\"$L3\"\n"])</script><script>self.__next_f.push([1,"4:I[47690,[],\"\"]\n6:I[77831,[],\"\"]\n7:I[29306,[\"823\",\"static/chunks/823-2ada48e2e6a5ab39.js\",\"931\",\"static/chunks/app/page-e16bcf8bdc356530.js\"],\"\"]\n8:I[5613,[],\"\"]\n9:I[31778,[],\"\"]\nb:I[48955,[],\"\"]\nc:[]\n"])</script><script>self.__next_f.push([1,"3:[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/ui/_next/static/css/a282d1bfd6ed4df8.css\",\"precedence\":\"next\",\"crossOrigin\":\"\"}]],[\"$\",\"$L4\",null,{\"buildId\":\"BNBzATtnAelV8BpmzRdfL\",\"assetPrefix\":\"/ui\",\"initialCanonicalUrl\":\"/\",\"initialTree\":[\"\",{\"children\":[\"__PAGE__\",{}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"__PAGE__\",{},[\"$L5\",[\"$\",\"$L6\",null,{\"propsForComponent\":{\"params\":{}},\"Component\":\"$7\",\"isStaticGeneration\":true}],null]]},[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"className\":\"__className_c23dc8\",\"children\":[\"$\",\"$L8\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"loadingScripts\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L9\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[],\"styles\":null}]}]}],null]],\"initialHead\":[false,\"$La\"],\"globalErrorComponent\":\"$b\",\"missingSlots\":\"$Wc\"}]]\n"])</script><script>self.__next_f.push([1,"a:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"LiteLLM Dashboard\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"LiteLLM Proxy Admin UI\"}],[\"$\",\"link\",\"4\",{\"rel\":\"icon\",\"href\":\"/ui/favicon.ico\",\"type\":\"image/x-icon\",\"sizes\":\"16x16\"}],[\"$\",\"meta\",\"5\",{\"name\":\"next-size-adjust\"}]]\n5:null\n"])</script><script>self.__next_f.push([1,""])</script></body></html>
|
||||
<!DOCTYPE html><html id="__next_error__"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/ui/_next/static/chunks/webpack-59f93936973f5f5a.js" crossorigin=""/><script src="/ui/_next/static/chunks/fd9d1056-bcf69420342937de.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/69-442a9c01c3fd20f9.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/main-app-096338c8e1915716.js" async="" crossorigin=""></script><title>LiteLLM Dashboard</title><meta name="description" content="LiteLLM Proxy Admin UI"/><link rel="icon" href="/ui/favicon.ico" type="image/x-icon" sizes="16x16"/><meta name="next-size-adjust"/><script src="/ui/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js" crossorigin="" noModule=""></script></head><body><script src="/ui/_next/static/chunks/webpack-59f93936973f5f5a.js" crossorigin="" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/ui/_next/static/media/c9a5bc6a7c948fb0-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n2:HL[\"/ui/_next/static/css/11cfce8bfdf6e8f1.css\",\"style\",{\"crossOrigin\":\"\"}]\n0:\"$L3\"\n"])</script><script>self.__next_f.push([1,"4:I[47690,[],\"\"]\n6:I[77831,[],\"\"]\n7:I[44488,[\"289\",\"static/chunks/289-04be6cb9636840d2.js\",\"931\",\"static/chunks/app/page-3dcbe3938de7e459.js\"],\"\"]\n8:I[5613,[],\"\"]\n9:I[31778,[],\"\"]\nb:I[48955,[],\"\"]\nc:[]\n"])</script><script>self.__next_f.push([1,"3:[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/ui/_next/static/css/11cfce8bfdf6e8f1.css\",\"precedence\":\"next\",\"crossOrigin\":\"\"}]],[\"$\",\"$L4\",null,{\"buildId\":\"G0uskIep5NGJbFqI3_DY1\",\"assetPrefix\":\"/ui\",\"initialCanonicalUrl\":\"/\",\"initialTree\":[\"\",{\"children\":[\"__PAGE__\",{}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"__PAGE__\",{},[\"$L5\",[\"$\",\"$L6\",null,{\"propsForComponent\":{\"params\":{}},\"Component\":\"$7\",\"isStaticGeneration\":true}],null]]},[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"className\":\"__className_c23dc8\",\"children\":[\"$\",\"$L8\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"loadingScripts\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L9\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[],\"styles\":null}]}]}],null]],\"initialHead\":[false,\"$La\"],\"globalErrorComponent\":\"$b\",\"missingSlots\":\"$Wc\"}]]\n"])</script><script>self.__next_f.push([1,"a:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"LiteLLM Dashboard\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"LiteLLM Proxy Admin UI\"}],[\"$\",\"link\",\"4\",{\"rel\":\"icon\",\"href\":\"/ui/favicon.ico\",\"type\":\"image/x-icon\",\"sizes\":\"16x16\"}],[\"$\",\"meta\",\"5\",{\"name\":\"next-size-adjust\"}]]\n5:null\n"])</script><script>self.__next_f.push([1,""])</script></body></html>
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
2:I[77831,[],""]
|
||||
3:I[29306,["823","static/chunks/823-2ada48e2e6a5ab39.js","931","static/chunks/app/page-e16bcf8bdc356530.js"],""]
|
||||
3:I[44488,["289","static/chunks/289-04be6cb9636840d2.js","931","static/chunks/app/page-3dcbe3938de7e459.js"],""]
|
||||
4:I[5613,[],""]
|
||||
5:I[31778,[],""]
|
||||
0:["BNBzATtnAelV8BpmzRdfL",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},["$L1",["$","$L2",null,{"propsForComponent":{"params":{}},"Component":"$3","isStaticGeneration":true}],null]]},[null,["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_c23dc8","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"loading":"$undefined","loadingStyles":"$undefined","loadingScripts":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[],"styles":null}]}]}],null]],[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/a282d1bfd6ed4df8.css","precedence":"next","crossOrigin":""}]],"$L6"]]]]
|
||||
0:["G0uskIep5NGJbFqI3_DY1",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},["$L1",["$","$L2",null,{"propsForComponent":{"params":{}},"Component":"$3","isStaticGeneration":true}],null]]},[null,["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_c23dc8","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"loading":"$undefined","loadingStyles":"$undefined","loadingScripts":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[],"styles":null}]}]}],null]],[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/11cfce8bfdf6e8f1.css","precedence":"next","crossOrigin":""}]],"$L6"]]]]
|
||||
6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/ui/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","meta","5",{"name":"next-size-adjust"}]]
|
||||
1:null
|
||||
|
|
|
|||
|
|
@ -74,7 +74,7 @@ async function generateModelResponse(
|
|||
}
|
||||
}
|
||||
} catch (error) {
|
||||
message.error(`Error occurred while generating model response. Please try again. Error: ${error}`);
|
||||
message.error(`Error occurred while generating model response. Please try again. Error: ${error}`, 20);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ import {
|
|||
} from "@tremor/react";
|
||||
import { TabPanel, TabPanels, TabGroup, TabList, Tab, TextInput, Icon } from "@tremor/react";
|
||||
import { Select, SelectItem, MultiSelect, MultiSelectItem } from "@tremor/react";
|
||||
import { modelInfoCall, userGetRequesedtModelsCall, modelMetricsCall, modelCreateCall, Model, modelCostMap, modelDeleteCall } from "./networking";
|
||||
import { modelInfoCall, userGetRequesedtModelsCall, modelMetricsCall, modelCreateCall, Model, modelCostMap, modelDeleteCall, healthCheckCall } from "./networking";
|
||||
import { BarChart } from "@tremor/react";
|
||||
import {
|
||||
Button as Button2,
|
||||
|
|
@ -36,6 +36,9 @@ import { Typography } from "antd";
|
|||
import TextArea from "antd/es/input/TextArea";
|
||||
import { InformationCircleIcon, PencilAltIcon, PencilIcon, StatusOnlineIcon, TrashIcon } from "@heroicons/react/outline";
|
||||
const { Title: Title2, Link } = Typography;
|
||||
import { UploadOutlined } from '@ant-design/icons';
|
||||
import type { UploadProps } from 'antd';
|
||||
import { Upload } from 'antd';
|
||||
|
||||
interface ModelDashboardProps {
|
||||
accessToken: string | null;
|
||||
|
|
@ -52,7 +55,8 @@ enum Providers {
|
|||
Anthropic = "Anthropic",
|
||||
Google_AI_Studio = "Gemini (Google AI Studio)",
|
||||
Bedrock = "Amazon Bedrock",
|
||||
OpenAI_Compatible = "OpenAI-Compatible Endpoints (Groq, Together AI, Mistral AI, etc.)"
|
||||
OpenAI_Compatible = "OpenAI-Compatible Endpoints (Groq, Together AI, Mistral AI, etc.)",
|
||||
Vertex_AI = "Vertex AI (Anthropic, Gemini, etc.)"
|
||||
}
|
||||
|
||||
const provider_map: Record <string, string> = {
|
||||
|
|
@ -61,9 +65,11 @@ const provider_map: Record <string, string> = {
|
|||
"Anthropic": "anthropic",
|
||||
"Google_AI_Studio": "gemini",
|
||||
"Bedrock": "bedrock",
|
||||
"OpenAI_Compatible": "openai"
|
||||
"OpenAI_Compatible": "openai",
|
||||
"Vertex_AI": "vertex_ai"
|
||||
};
|
||||
|
||||
|
||||
const ModelDashboard: React.FC<ModelDashboardProps> = ({
|
||||
accessToken,
|
||||
token,
|
||||
|
|
@ -78,9 +84,41 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
|
|||
|
||||
const [providerModels, setProviderModels] = useState<Array<string>>([]); // Explicitly typing providerModels as a string array
|
||||
|
||||
const providers: Providers[] = [Providers.OpenAI, Providers.Azure, Providers.Anthropic, Providers.Google_AI_Studio, Providers.Bedrock, Providers.OpenAI_Compatible]
|
||||
const providers = Object.values(Providers).filter(key => isNaN(Number(key)));
|
||||
|
||||
|
||||
const [selectedProvider, setSelectedProvider] = useState<String>("OpenAI");
|
||||
const [healthCheckResponse, setHealthCheckResponse] = useState<string>('');
|
||||
|
||||
|
||||
const props: UploadProps = {
|
||||
name: 'file',
|
||||
accept: '.json',
|
||||
beforeUpload: file => {
|
||||
if (file.type === 'application/json') {
|
||||
const reader = new FileReader();
|
||||
reader.onload = (e) => {
|
||||
if (e.target) {
|
||||
const jsonStr = e.target.result as string;
|
||||
form.setFieldsValue({ vertex_credentials: jsonStr });
|
||||
}
|
||||
};
|
||||
reader.readAsText(file);
|
||||
}
|
||||
// Prevent upload
|
||||
return false;
|
||||
},
|
||||
onChange(info) {
|
||||
if (info.file.status !== 'uploading') {
|
||||
console.log(info.file, info.fileList);
|
||||
}
|
||||
if (info.file.status === 'done') {
|
||||
message.success(`${info.file.name} file uploaded successfully`);
|
||||
} else if (info.file.status === 'error') {
|
||||
message.error(`${info.file.name} file upload failed.`);
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
if (!accessToken || !token || !userRole || !userID) {
|
||||
|
|
@ -230,22 +268,43 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
|
|||
|
||||
const setProviderModelsFn = (provider: string) => {
|
||||
console.log(`received provider string: ${provider}`)
|
||||
const providerEnumValue = Providers[provider as keyof typeof Providers];
|
||||
console.log(`received providerEnumValue: ${providerEnumValue}`)
|
||||
const mappingResult = provider_map[providerEnumValue]; // Get the corresponding value from the mapping
|
||||
console.log(`mappingResult: ${mappingResult}`)
|
||||
let _providerModels: Array<string> = []
|
||||
if (typeof modelMap === 'object') {
|
||||
Object.entries(modelMap).forEach(([key, value]) => {
|
||||
if (value !== null && typeof value === 'object' && "litellm_provider" in value && value["litellm_provider"] === mappingResult) {
|
||||
_providerModels.push(key);
|
||||
}
|
||||
});
|
||||
const providerKey = Object.keys(Providers).find(key => (Providers as {[index: string]: any})[key] === provider);
|
||||
|
||||
if (providerKey) {
|
||||
const mappingResult = provider_map[providerKey]; // Get the corresponding value from the mapping
|
||||
console.log(`mappingResult: ${mappingResult}`)
|
||||
let _providerModels: Array<string> = []
|
||||
if (typeof modelMap === 'object') {
|
||||
Object.entries(modelMap).forEach(([key, value]) => {
|
||||
if (
|
||||
value !== null
|
||||
&& typeof value === 'object'
|
||||
&& "litellm_provider" in (value as object)
|
||||
&& (
|
||||
(value as any)["litellm_provider"] === mappingResult
|
||||
|| (value as any)["litellm_provider"].includes(mappingResult)
|
||||
)) {
|
||||
_providerModels.push(key);
|
||||
}
|
||||
});
|
||||
}
|
||||
setProviderModels(_providerModels)
|
||||
console.log(`providerModels: ${providerModels}`);
|
||||
}
|
||||
setProviderModels(_providerModels)
|
||||
console.log(`providerModels: ${providerModels}`);
|
||||
}
|
||||
|
||||
const runHealthCheck = async () => {
|
||||
try {
|
||||
message.info('Running health check...');
|
||||
setHealthCheckResponse('');
|
||||
const response = await healthCheckCall(accessToken);
|
||||
setHealthCheckResponse(response);
|
||||
} catch (error) {
|
||||
console.error('Error running health check:', error);
|
||||
setHealthCheckResponse('Error running health check');
|
||||
}
|
||||
};
|
||||
|
||||
const handleSubmit = async (formValues: Record<string, any>) => {
|
||||
try {
|
||||
/**
|
||||
|
|
@ -294,7 +353,7 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
|
|||
litellmExtraParams = JSON.parse(value);
|
||||
}
|
||||
catch (error) {
|
||||
message.error("Failed to parse LiteLLM Extra Params: " + error);
|
||||
message.error("Failed to parse LiteLLM Extra Params: " + error, 20);
|
||||
throw new Error("Failed to parse litellm_extra_params: " + error);
|
||||
}
|
||||
for (const [key, value] of Object.entries(litellmExtraParams)) {
|
||||
|
|
@ -330,10 +389,24 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
|
|||
|
||||
|
||||
} catch (error) {
|
||||
message.error("Failed to create model: " + error);
|
||||
message.error("Failed to create model: " + error, 20);
|
||||
}
|
||||
}
|
||||
|
||||
const getPlaceholder = (selectedProvider: string): string => {
|
||||
if (selectedProvider === Providers.Vertex_AI) {
|
||||
return 'gemini-pro';
|
||||
} else if (selectedProvider == Providers.Anthropic) {
|
||||
return 'claude-3-opus'
|
||||
} else if (selectedProvider == Providers.Bedrock) {
|
||||
return 'claude-3-opus'
|
||||
} else if (selectedProvider == Providers.Google_AI_Studio) {
|
||||
return 'gemini-pro'
|
||||
} else {
|
||||
return 'gpt-3.5-turbo';
|
||||
}
|
||||
};
|
||||
|
||||
const handleOk = () => {
|
||||
form
|
||||
.validateFields()
|
||||
|
|
@ -354,6 +427,7 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
|
|||
<TabList className="mt-2">
|
||||
<Tab>All Models</Tab>
|
||||
<Tab>Add Model</Tab>
|
||||
<Tab><pre>/health Models</pre></Tab>
|
||||
</TabList>
|
||||
|
||||
<TabPanels>
|
||||
|
|
@ -439,9 +513,6 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
|
|||
</Grid>
|
||||
</TabPanel>
|
||||
<TabPanel className="h-full">
|
||||
{/* <Card className="mx-auto max-w-lg flex flex-col h-[60vh] space-between">
|
||||
|
||||
</Card> */}
|
||||
<Title2 level={2}>Add new model</Title2>
|
||||
<Card>
|
||||
<Form
|
||||
|
|
@ -469,7 +540,7 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
|
|||
</Select>
|
||||
</Form.Item>
|
||||
<Form.Item rules={[{ required: true, message: 'Required' }]} label="Public Model Name" name="model_name" tooltip="Model name your users will pass in. Also used for load-balancing, LiteLLM will load balance between all models with this public name." className="mb-0">
|
||||
<TextInput placeholder="gpt-3.5-turbo"/>
|
||||
<TextInput placeholder={getPlaceholder(selectedProvider.toString())}/>
|
||||
</Form.Item>
|
||||
<Row>
|
||||
<Col span={10}></Col>
|
||||
|
|
@ -492,9 +563,9 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
|
|||
</Form.Item>
|
||||
<Row>
|
||||
<Col span={10}></Col>
|
||||
<Col span={10}><Text className="mb-3 mt-1">Actual model name used for making<Link href="https://docs.litellm.ai/docs/providers" target="_blank">litellm.completion() call</Link>.We'll<Link href="https://docs.litellm.ai/docs/proxy/reliability#step-1---set-deployments-on-config" target="_blank">loadbalance</Link> models with the same 'public name'</Text></Col></Row>
|
||||
<Col span={10}><Text className="mb-3 mt-1">Actual model name used for making <Link href="https://docs.litellm.ai/docs/providers" target="_blank">litellm.completion() call</Link>. We'll <Link href="https://docs.litellm.ai/docs/proxy/reliability#step-1---set-deployments-on-config" target="_blank">loadbalance</Link> models with the same 'public name'</Text></Col></Row>
|
||||
{
|
||||
selectedProvider != Providers.Bedrock && <Form.Item
|
||||
selectedProvider != Providers.Bedrock && selectedProvider != Providers.Vertex_AI && <Form.Item
|
||||
rules={[{ required: true, message: 'Required' }]}
|
||||
label="API Key"
|
||||
name="api_key"
|
||||
|
|
@ -510,6 +581,32 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
|
|||
<TextInput placeholder="[OPTIONAL] my-unique-org"/>
|
||||
</Form.Item>
|
||||
}
|
||||
{
|
||||
selectedProvider == Providers.Vertex_AI && <Form.Item rules={[{ required: true, message: 'Required' }]}
|
||||
label="Vertex Project"
|
||||
name="vertex_project"><TextInput placeholder="adroit-cadet-1234.."/></Form.Item>
|
||||
}
|
||||
{
|
||||
selectedProvider == Providers.Vertex_AI && <Form.Item rules={[{ required: true, message: 'Required' }]}
|
||||
label="Vertex Location"
|
||||
name="vertex_location"><TextInput placeholder="us-east-1"/></Form.Item>
|
||||
}
|
||||
{
|
||||
selectedProvider == Providers.Vertex_AI && <Form.Item rules={[{ required: true, message: 'Required' }]}
|
||||
label="Vertex Credentials"
|
||||
name="vertex_credentials"
|
||||
className="mb-0">
|
||||
<Upload {...props}>
|
||||
<Button2 icon={<UploadOutlined />}>Click to Upload</Button2>
|
||||
</Upload>
|
||||
</Form.Item>
|
||||
}
|
||||
{
|
||||
selectedProvider == Providers.Vertex_AI && <Row>
|
||||
<Col span={10}></Col>
|
||||
<Col span={10}><Text className="mb-3 mt-1">Give litellm a gcp service account(.json file), so it can make the relevant calls</Text></Col></Row>
|
||||
|
||||
}
|
||||
{
|
||||
(selectedProvider == Providers.Azure || selectedProvider == Providers.OpenAI_Compatible) && <Form.Item
|
||||
rules={[{ required: true, message: 'Required' }]}
|
||||
|
|
@ -593,6 +690,17 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
|
|||
</Form>
|
||||
</Card>
|
||||
</TabPanel>
|
||||
<TabPanel>
|
||||
<Card>
|
||||
<Text>`/health` will run a very small request through your models configured on litellm</Text>
|
||||
|
||||
<Button onClick={runHealthCheck}>Run `/health`</Button>
|
||||
{healthCheckResponse && (
|
||||
<pre>{JSON.stringify(healthCheckResponse, null, 2)}</pre>
|
||||
)}
|
||||
|
||||
</Card>
|
||||
</TabPanel>
|
||||
</TabPanels>
|
||||
</TabGroup>
|
||||
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ export const modelCreateCall = async (
|
|||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error("Failed to create key: " + errorData);
|
||||
message.error("Failed to create key: " + errorData, 20);
|
||||
console.error("Error response from the server:", errorData);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
|
@ -78,7 +78,7 @@ export const modelDeleteCall = async (
|
|||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error("Failed to create key: " + errorData);
|
||||
message.error("Failed to create key: " + errorData, 20);
|
||||
console.error("Error response from the server:", errorData);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
|
@ -120,7 +120,7 @@ export const keyCreateCall = async (
|
|||
try {
|
||||
formValues.metadata = JSON.parse(formValues.metadata);
|
||||
} catch (error) {
|
||||
message.error("Failed to parse metadata: " + error);
|
||||
message.error("Failed to parse metadata: " + error, 20);
|
||||
throw new Error("Failed to parse metadata: " + error);
|
||||
}
|
||||
}
|
||||
|
|
@ -141,7 +141,7 @@ export const keyCreateCall = async (
|
|||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error("Failed to create key: " + errorData);
|
||||
message.error("Failed to create key: " + errorData, 20);
|
||||
console.error("Error response from the server:", errorData);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
|
@ -183,7 +183,7 @@ export const userCreateCall = async (
|
|||
try {
|
||||
formValues.metadata = JSON.parse(formValues.metadata);
|
||||
} catch (error) {
|
||||
message.error("Failed to parse metadata: " + error);
|
||||
message.error("Failed to parse metadata: " + error, 20);
|
||||
throw new Error("Failed to parse metadata: " + error);
|
||||
}
|
||||
}
|
||||
|
|
@ -204,7 +204,7 @@ export const userCreateCall = async (
|
|||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error("Failed to create key: " + errorData);
|
||||
message.error("Failed to create key: " + errorData, 20);
|
||||
console.error("Error response from the server:", errorData);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
|
@ -237,7 +237,7 @@ export const keyDeleteCall = async (accessToken: String, user_key: String) => {
|
|||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error("Failed to delete key: " + errorData);
|
||||
message.error("Failed to delete key: " + errorData, 20);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
||||
|
|
@ -269,7 +269,7 @@ export const teamDeleteCall = async (accessToken: String, teamID: String) => {
|
|||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error("Failed to delete team: " + errorData);
|
||||
message.error("Failed to delete team: " + errorData, 20);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
const data = await response.json();
|
||||
|
|
@ -297,7 +297,7 @@ export const userInfoCall = async (
|
|||
url = `${url}?user_id=${userID}`;
|
||||
}
|
||||
console.log("in userInfoCall viewAll=", viewAll);
|
||||
if (viewAll && page && page_size) {
|
||||
if (viewAll && page_size && (page != null) && (page != undefined)) {
|
||||
url = `${url}?view_all=true&page=${page}&page_size=${page_size}`;
|
||||
}
|
||||
//message.info("Requesting user data");
|
||||
|
|
@ -311,7 +311,7 @@ export const userInfoCall = async (
|
|||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error(errorData);
|
||||
message.error(errorData, 20);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
||||
|
|
@ -346,7 +346,7 @@ export const getTotalSpendCall = async (
|
|||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error(errorData);
|
||||
message.error(errorData, 20);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
||||
|
|
@ -381,7 +381,7 @@ export const modelInfoCall = async (
|
|||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error(errorData);
|
||||
message.error(errorData, 20);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
||||
|
|
@ -418,7 +418,7 @@ export const modelMetricsCall = async (
|
|||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error(errorData);
|
||||
message.error(errorData, 20);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
const data = await response.json();
|
||||
|
|
@ -453,7 +453,7 @@ export const modelAvailableCall = async (
|
|||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error(errorData);
|
||||
message.error(errorData, 20);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
||||
|
|
@ -482,7 +482,7 @@ export const keySpendLogsCall = async (accessToken: String, token: String) => {
|
|||
});
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error(errorData);
|
||||
message.error(errorData, 20);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
||||
|
|
@ -511,7 +511,7 @@ export const teamSpendLogsCall = async (accessToken: String) => {
|
|||
});
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error(errorData);
|
||||
message.error(errorData, 20);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
||||
|
|
@ -540,7 +540,7 @@ export const tagsSpendLogsCall = async (accessToken: String) => {
|
|||
});
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error(errorData);
|
||||
message.error(errorData, 20);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
||||
|
|
@ -580,7 +580,7 @@ export const userSpendLogsCall = async (
|
|||
});
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error(errorData);
|
||||
message.error(errorData, 20);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
||||
|
|
@ -610,7 +610,7 @@ export const adminSpendLogsCall = async (accessToken: String) => {
|
|||
});
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error(errorData);
|
||||
message.error(errorData, 20);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
||||
|
|
@ -640,7 +640,7 @@ export const adminTopKeysCall = async (accessToken: String) => {
|
|||
});
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error(errorData);
|
||||
message.error(errorData, 20);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
||||
|
|
@ -692,7 +692,7 @@ export const adminTopEndUsersCall = async (
|
|||
const response = await fetch(url, requestOptions);
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error(errorData);
|
||||
message.error(errorData, 20);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
||||
|
|
@ -722,7 +722,7 @@ export const adminTopModelsCall = async (accessToken: String) => {
|
|||
});
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error(errorData);
|
||||
message.error(errorData, 20);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
||||
|
|
@ -753,7 +753,7 @@ export const keyInfoCall = async (accessToken: String, keys: String[]) => {
|
|||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error(errorData);
|
||||
message.error(errorData, 20);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
||||
|
|
@ -779,7 +779,7 @@ export const spendUsersCall = async (accessToken: String, userID: String) => {
|
|||
});
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error(errorData);
|
||||
message.error(errorData, 20);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
||||
|
|
@ -817,7 +817,7 @@ export const userRequestModelCall = async (
|
|||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error("Failed to delete key: " + errorData);
|
||||
message.error("Failed to delete key: " + errorData, 20);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
const data = await response.json();
|
||||
|
|
@ -847,7 +847,7 @@ export const userGetRequesedtModelsCall = async (accessToken: String) => {
|
|||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error("Failed to delete key: " + errorData);
|
||||
message.error("Failed to delete key: " + errorData, 20);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
const data = await response.json();
|
||||
|
|
@ -887,7 +887,7 @@ export const userGetAllUsersCall = async (
|
|||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error("Failed to delete key: " + errorData);
|
||||
message.error("Failed to delete key: " + errorData, 20);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
const data = await response.json();
|
||||
|
|
@ -922,7 +922,7 @@ export const teamCreateCall = async (
|
|||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error("Failed to create key: " + errorData);
|
||||
message.error("Failed to create key: " + errorData, 20);
|
||||
console.error("Error response from the server:", errorData);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
|
@ -959,7 +959,7 @@ export const keyUpdateCall = async (
|
|||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error("Failed to update key: " + errorData);
|
||||
message.error("Failed to update key: " + errorData, 20);
|
||||
console.error("Error response from the server:", errorData);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
|
@ -994,7 +994,7 @@ export const teamUpdateCall = async (
|
|||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error("Failed to update team: " + errorData);
|
||||
message.error("Failed to update team: " + errorData, 20);
|
||||
console.error("Error response from the server:", errorData);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
|
@ -1039,7 +1039,7 @@ export const teamMemberAddCall = async (
|
|||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error("Failed to create key: " + errorData);
|
||||
message.error("Failed to create key: " + errorData, 20);
|
||||
console.error("Error response from the server:", errorData);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
|
@ -1079,7 +1079,7 @@ export const userUpdateUserCall = async (
|
|||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error("Failed to create key: " + errorData);
|
||||
message.error("Failed to create key: " + errorData, 20);
|
||||
console.error("Error response from the server:", errorData);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
|
@ -1119,7 +1119,7 @@ export const PredictedSpendLogsCall = async (
|
|||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error(errorData);
|
||||
message.error(errorData, 20);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
||||
|
|
@ -1172,6 +1172,42 @@ export const slackBudgetAlertsHealthCheck = async (accessToken: String) => {
|
|||
|
||||
|
||||
|
||||
export const serviceHealthCheck= async (accessToken: String, service: String) => {
|
||||
try {
|
||||
let url = proxyBaseUrl
|
||||
? `${proxyBaseUrl}/health/services?service=${service}`
|
||||
: `/health/services?service=${service}`;
|
||||
|
||||
console.log("Checking Slack Budget Alerts service health");
|
||||
|
||||
const response = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error(`Failed ${service} service health check ` + errorData);
|
||||
// throw error with message
|
||||
throw new Error(errorData);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
message.success(`Test request to ${service} made - check logs on ${service} dashboard!`);
|
||||
// You can add additional logic here based on the response if needed
|
||||
return data;
|
||||
} catch (error) {
|
||||
console.error("Failed to perform health check:", error);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
|
||||
|
||||
export const getCallbacksCall = async (
|
||||
accessToken: String,
|
||||
userID: String,
|
||||
|
|
@ -1194,7 +1230,7 @@ export const getCallbacksCall = async (
|
|||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error(errorData);
|
||||
message.error(errorData, 20);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
||||
|
|
@ -1237,7 +1273,7 @@ export const setCallbacksCall = async (
|
|||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error(errorData);
|
||||
message.error(errorData, 20);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
||||
|
|
@ -1251,4 +1287,39 @@ export const setCallbacksCall = async (
|
|||
}
|
||||
};
|
||||
|
||||
export const healthCheckCall = async (
|
||||
accessToken: String,
|
||||
) => {
|
||||
/**
|
||||
* Get all the models user has access to
|
||||
*/
|
||||
try {
|
||||
let url = proxyBaseUrl ? `${proxyBaseUrl}/health` : `/health`;
|
||||
|
||||
//message.info("Requesting model data");
|
||||
const response = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error(errorData);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
//message.info("Received model data");
|
||||
return data;
|
||||
// Handle success - you might want to update some state or UI based on the created key
|
||||
} catch (error) {
|
||||
console.error("Failed to call /health:", error);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -14,10 +14,11 @@ import {
|
|||
Text,
|
||||
Grid,
|
||||
Button,
|
||||
TextInput,
|
||||
Col,
|
||||
} from "@tremor/react";
|
||||
import { getCallbacksCall, setCallbacksCall } from "./networking";
|
||||
import { Modal, Form, Input, Select, Button as Button2 } from "antd";
|
||||
import { getCallbacksCall, setCallbacksCall, serviceHealthCheck } from "./networking";
|
||||
import { Modal, Form, Input, Select, Button as Button2, message } from "antd";
|
||||
import StaticGenerationSearchParamsBailoutProvider from "next/dist/client/components/static-generation-searchparams-bailout-provider";
|
||||
|
||||
interface SettingsPageProps {
|
||||
|
|
@ -31,7 +32,7 @@ const Settings: React.FC<SettingsPageProps> = ({
|
|||
userRole,
|
||||
userID,
|
||||
}) => {
|
||||
const [callbacks, setCallbacks] = useState<string[]>([]);
|
||||
const [callbacks, setCallbacks] = useState<any[]>([]);
|
||||
const [isModalVisible, setIsModalVisible] = useState(false);
|
||||
const [form] = Form.useForm();
|
||||
const [selectedCallback, setSelectedCallback] = useState<string | null>(null);
|
||||
|
|
@ -43,8 +44,7 @@ const Settings: React.FC<SettingsPageProps> = ({
|
|||
getCallbacksCall(accessToken, userID, userRole).then((data) => {
|
||||
console.log("callbacks", data);
|
||||
let callbacks_data = data.data;
|
||||
let callback_names = callbacks_data.success_callback; // ["callback1", "callback2"]
|
||||
setCallbacks(callback_names);
|
||||
setCallbacks(callbacks_data);
|
||||
});
|
||||
}, [accessToken, userRole, userID]);
|
||||
|
||||
|
|
@ -59,6 +59,28 @@ const Settings: React.FC<SettingsPageProps> = ({
|
|||
setSelectedCallback(null);
|
||||
};
|
||||
|
||||
const handleSaveChanges = (callback: any) => {
|
||||
if (!accessToken) {
|
||||
return;
|
||||
}
|
||||
|
||||
const updatedVariables = Object.fromEntries(
|
||||
Object.entries(callback.variables).map(([key, value]) => [key, (document.querySelector(`input[name="${key}"]`) as HTMLInputElement)?.value || value])
|
||||
);
|
||||
|
||||
const payload = {
|
||||
environment_variables: updatedVariables,
|
||||
};
|
||||
|
||||
try {
|
||||
setCallbacksCall(accessToken, payload);
|
||||
} catch (error) {
|
||||
message.error("Failed to update callback: " + error, 20);
|
||||
}
|
||||
|
||||
message.success("Callback updated successfully");
|
||||
};
|
||||
|
||||
const handleOk = () => {
|
||||
if (!accessToken) {
|
||||
return;
|
||||
|
|
@ -111,36 +133,59 @@ const Settings: React.FC<SettingsPageProps> = ({
|
|||
setSelectedCallback(value);
|
||||
};
|
||||
|
||||
if (!accessToken) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="w-full mx-4">
|
||||
<Grid numItems={1} className="gap-2 p-8 h-[75vh] w-full mt-2">
|
||||
<Card className="h-[15vh]">
|
||||
<Grid numItems={2} className="mt-2">
|
||||
<Col>
|
||||
<Title>Logging Callbacks</Title>
|
||||
</Col>
|
||||
<Col>
|
||||
<div>
|
||||
{!callbacks ? (
|
||||
<Badge color={"red"}>None</Badge>
|
||||
) : callbacks.length === 0 ? (
|
||||
<Badge>None</Badge>
|
||||
) : (
|
||||
callbacks.map((callback, index) => (
|
||||
<Badge key={index} color={"sky"}>
|
||||
{callback}
|
||||
</Badge>
|
||||
))
|
||||
)}
|
||||
</div>
|
||||
</Col>
|
||||
</Grid>
|
||||
<Col>
|
||||
<Grid numItems={1} className="gap-2 p-8 w-full mt-2">
|
||||
<Title>Logging Callbacks</Title>
|
||||
<Card >
|
||||
<Table>
|
||||
<TableHead>
|
||||
<TableRow>
|
||||
<TableHeaderCell>Callback</TableHeaderCell>
|
||||
<TableHeaderCell>Callback Env Vars</TableHeaderCell>
|
||||
</TableRow>
|
||||
</TableHead>
|
||||
<TableBody>
|
||||
{callbacks.map((callback, index) => (
|
||||
<TableRow key={index}>
|
||||
<TableCell>
|
||||
<Badge color="emerald">{callback.name}</Badge>
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
<ul>
|
||||
{Object.entries(callback.variables).map(([key, value]) => (
|
||||
<li key={key}>
|
||||
<Text className="mt-2">{key}</Text>
|
||||
{key === "LANGFUSE_HOST" ? (
|
||||
<p>default value=https://cloud.langfuse.com</p>
|
||||
) : (
|
||||
<div></div>
|
||||
)}
|
||||
<TextInput name={key} defaultValue={value as string} type="password" />
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
<Button className="mt-2" onClick={() => handleSaveChanges(callback)}>
|
||||
Save Changes
|
||||
</Button>
|
||||
<Button onClick={() => serviceHealthCheck(accessToken, callback.name)} className="mx-2">
|
||||
Test Callback
|
||||
</Button>
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
</TableBody>
|
||||
</Table>
|
||||
<Button size="xs" className="mt-2" onClick={handleAddCallback}>
|
||||
Add Callback
|
||||
</Button>
|
||||
</Col>
|
||||
|
||||
</Card>
|
||||
|
||||
</Grid>
|
||||
|
||||
<Modal
|
||||
|
|
|
|||
|
|
@ -290,7 +290,7 @@ const handleEditSubmit = async (formValues: Record<string, any>) => {
|
|||
}
|
||||
} catch (error) {
|
||||
console.error("Error creating the team:", error);
|
||||
message.error("Error creating the team: " + error);
|
||||
message.error("Error creating the team: " + error, 20);
|
||||
}
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ const ViewUserDashboard: React.FC<ViewUserDashboardProps> = ({
|
|||
}) => {
|
||||
const [userData, setUserData] = useState<null | any[]>(null);
|
||||
const [endUsers, setEndUsers] = useState<null | any[]>(null);
|
||||
const [currentPage, setCurrentPage] = useState(1);
|
||||
const [currentPage, setCurrentPage] = useState(0);
|
||||
const defaultPageSize = 25;
|
||||
|
||||
useEffect(() => {
|
||||
|
|
@ -113,19 +113,17 @@ const ViewUserDashboard: React.FC<ViewUserDashboardProps> = ({
|
|||
function renderPagination() {
|
||||
if (!userData) return null;
|
||||
|
||||
// const totalPages = Math.ceil(userData.length / defaultPageSize);
|
||||
const startItem = (currentPage - 1) * defaultPageSize + 1;
|
||||
const endItem = Math.min(currentPage * defaultPageSize, userData.length);
|
||||
const totalPages = Math.ceil(userData.length / defaultPageSize);
|
||||
|
||||
return (
|
||||
<div className="flex justify-between items-center">
|
||||
<div>
|
||||
Showing {startItem} – {endItem} of {userData.length}
|
||||
Showing Page {currentPage+1} of {totalPages}
|
||||
</div>
|
||||
<div className="flex">
|
||||
<button
|
||||
className="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded-l focus:outline-none"
|
||||
disabled={currentPage === 1}
|
||||
disabled={currentPage === 0}
|
||||
onClick={() => setCurrentPage(currentPage - 1)}
|
||||
>
|
||||
← Prev
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue